diff --git a/.clang-format b/.clang-format index c8b9672dc7d..d8f273702c8 100644 --- a/.clang-format +++ b/.clang-format @@ -12,6 +12,7 @@ BraceWrapping: AfterUnion: true BeforeCatch: true BeforeElse: true + BeforeLambdaBody: true IndentBraces: false BreakConstructorInitializersBeforeComma: false Cpp11BracedListStyle: true diff --git a/.clang-tidy b/.clang-tidy index ecb8ac6dcbf..ddd0ee6d911 100644 --- a/.clang-tidy +++ b/.clang-tidy @@ -142,6 +142,7 @@ Checks: '-*, clang-analyzer-cplusplus.PlacementNewChecker, clang-analyzer-cplusplus.SelfAssignment, clang-analyzer-deadcode.DeadStores, + clang-analyzer-cplusplus.Move, clang-analyzer-optin.cplusplus.VirtualCall, clang-analyzer-security.insecureAPI.UncheckedReturn, clang-analyzer-security.insecureAPI.bcmp, diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index 91b9ea5bf3d..5816a58081d 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -86,6 +86,7 @@ jobs: StyleCheck: needs: DockerHubPush runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | @@ -93,6 +94,8 @@ jobs: TEMP_PATH=${{ runner.temp }}/style_check EOF - name: Download changed images + # even if artifact does not exist, e.g. on `do not test` label or failed Docker job + continue-on-error: true uses: actions/download-artifact@v2 with: name: changed_images @@ -1062,6 +1065,41 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseS3: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_s3_storage + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, s3 storage, actions) + REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] @@ -2841,6 +2879,7 @@ jobs: - FunctionalStatefulTestDebug - FunctionalStatefulTestRelease - FunctionalStatefulTestReleaseDatabaseOrdinary + - FunctionalStatelessTestReleaseS3 - FunctionalStatefulTestAarch64 - FunctionalStatefulTestAsan - FunctionalStatefulTestTsan diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 00000000000..a172947b2fc --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,73 @@ +name: NightlyBuilds + +env: + # Force the stdout and stderr streams to be unbuffered + PYTHONUNBUFFERED: 1 + +"on": + schedule: + - cron: '13 3 * * *' + +jobs: + DockerHubPushAarch64: + runs-on: [self-hosted, style-checker-aarch64] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix aarch64 --all + - name: Upload images files to artifacts + uses: actions/upload-artifact@v2 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }}/docker_images_check/changed_images_aarch64.json + DockerHubPushAmd64: + runs-on: [self-hosted, style-checker] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_images_check.py --suffix amd64 --all + - name: Upload images files to artifacts + uses: actions/upload-artifact@v2 + with: + name: changed_images_amd64 + path: ${{ runner.temp }}/docker_images_check/changed_images_amd64.json + DockerHubPush: + needs: [DockerHubPushAmd64, DockerHubPushAarch64] + runs-on: [self-hosted, style-checker] + steps: + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Download changed aarch64 images + uses: actions/download-artifact@v2 + with: + name: changed_images_aarch64 + path: ${{ runner.temp }} + - name: Download changed amd64 images + uses: actions/download-artifact@v2 + with: + name: changed_images_amd64 + path: ${{ runner.temp }} + - name: Images check + run: | + cd "$GITHUB_WORKSPACE/tests/ci" + python3 docker_manifests_merge.py --suffix amd64 --suffix aarch64 + - name: Upload images files to artifacts + uses: actions/upload-artifact@v2 + with: + name: changed_images + path: ${{ runner.temp }}/changed_images.json diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index cd8517de8fe..960e24d693c 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -111,6 +111,7 @@ jobs: StyleCheck: needs: DockerHubPush runs-on: [self-hosted, style-checker] + if: ${{ success() || failure() }} steps: - name: Set envs run: | @@ -118,6 +119,8 @@ jobs: TEMP_PATH=${{ runner.temp }}/style_check EOF - name: Download changed images + # even if artifact does not exist, e.g. on `do not test` label or failed Docker job + continue-on-error: true uses: actions/download-artifact@v2 with: name: changed_images @@ -1212,6 +1215,41 @@ jobs: docker kill "$(docker ps -q)" ||: docker rm -f "$(docker ps -a -q)" ||: sudo rm -fr "$TEMP_PATH" + FunctionalStatelessTestReleaseS3: + needs: [BuilderDebRelease] + runs-on: [self-hosted, func-tester] + steps: + - name: Set envs + run: | + cat >> "$GITHUB_ENV" << 'EOF' + TEMP_PATH=${{runner.temp}}/stateless_s3_storage + REPORTS_PATH=${{runner.temp}}/reports_dir + CHECK_NAME=Stateless tests (release, s3 storage, actions) + REPO_COPY=${{runner.temp}}/stateless_s3_storage/ClickHouse + KILL_TIMEOUT=10800 + EOF + - name: Download json reports + uses: actions/download-artifact@v2 + with: + path: ${{ env.REPORTS_PATH }} + - name: Clear repository + run: | + sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE" + - name: Check out repository code + uses: actions/checkout@v2 + - name: Functional test + run: | + sudo rm -fr "$TEMP_PATH" + mkdir -p "$TEMP_PATH" + cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH" + cd "$REPO_COPY/tests/ci" + python3 functional_test_check.py "$CHECK_NAME" "$KILL_TIMEOUT" + - name: Cleanup + if: always() + run: | + docker kill "$(docker ps -q)" ||: + docker rm -f "$(docker ps -a -q)" ||: + sudo rm -fr "$TEMP_PATH" FunctionalStatelessTestAarch64: needs: [BuilderDebAarch64] runs-on: [self-hosted, func-tester-aarch64] @@ -3034,6 +3072,7 @@ jobs: - FunctionalStatefulTestTsan - FunctionalStatefulTestMsan - FunctionalStatefulTestUBsan + - FunctionalStatelessTestReleaseS3 - StressTestDebug - StressTestAsan - StressTestTsan diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 77bc285196c..46e36c846d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -22,7 +22,6 @@ jobs: - name: Check out repository code uses: actions/checkout@v2 - name: Download packages and push to Artifactory - env: run: | rm -rf "$TEMP_PATH" && mkdir -p "$REPO_COPY" cp -r "$GITHUB_WORKSPACE" "$REPO_COPY" diff --git a/.github/workflows/tags_stable.yml b/.github/workflows/tags_stable.yml new file mode 100644 index 00000000000..30b6bfb027e --- /dev/null +++ b/.github/workflows/tags_stable.yml @@ -0,0 +1,38 @@ +name: TagsStableWorkflow +# - Gets artifacts from S3 +# - Sends it to JFROG Artifactory +# - Adds them to the release assets + +on: # yamllint disable-line rule:truthy + push: + tags: + - 'v*-stable' + - 'v*-lts' + + +jobs: + UpdateVersions: + runs-on: [self-hosted, style-checker] + steps: + - name: Get tag name + run: echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV" + - name: Check out repository code + uses: actions/checkout@v2 + with: + ref: master + - name: Generate versions + run: | + git fetch --tags + ./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv + - name: Create Pull Request + uses: peter-evans/create-pull-request@v3 + with: + commit-message: Update version_date.tsv after ${{ env.GITHUB_TAG }} + branch: auto/${{ env.GITHUB_TAG }} + delete-branch: true + title: Update version_date.tsv after ${{ env.GITHUB_TAG }} + body: | + Update version_date.tsv after ${{ env.GITHUB_TAG }} + + Changelog category (leave one): + - Not for changelog (changelog entry is not required) diff --git a/.gitmodules b/.gitmodules index ed023ab348b..91f4ddb2007 100644 --- a/.gitmodules +++ b/.gitmodules @@ -259,3 +259,6 @@ [submodule "contrib/azure"] path = contrib/azure url = https://github.com/ClickHouse-Extras/azure-sdk-for-cpp.git +[submodule "contrib/minizip-ng"] + path = contrib/minizip-ng + url = https://github.com/zlib-ng/minizip-ng diff --git a/.potato.yml b/.potato.yml deleted file mode 100644 index 7cb87c58bd1..00000000000 --- a/.potato.yml +++ /dev/null @@ -1,27 +0,0 @@ -# This is the configuration file with settings for Potato. -# Potato is an internal Yandex technology that allows us to sync internal [Yandex.Tracker](https://yandex.com/tracker/) and GitHub. - -# For all PRs where documentation is needed, just add a 'pr-feature' label and we will include it into documentation sprints. - -# The project name. -name: clickhouse -# Object handlers defines which handlers we use. -handlers: - # The handler for creating an Yandex.Tracker issue. - - name: issue-create - params: - triggers: - # The trigger for creating the Yandex.Tracker issue. When the specified event occurs, it transfers PR data to Yandex.Tracker. - github:pullRequest:labeled: - data: - # The Yandex.Tracker queue to create the issue in. Each issue in Tracker belongs to one of the project queues. - queue: CLICKHOUSEDOCS - # The issue title. - summary: '[Potato] Pull Request #{{pullRequest.number}}' - # The issue description. - description: > - {{pullRequest.description}} - - Ссылка на Pull Request: {{pullRequest.webUrl}} - # The condition for creating the Yandex.Tracker issue. - condition: eventPayload.labels.filter(label => ['pr-feature'].includes(label.name)).length diff --git a/CMakeLists.txt b/CMakeLists.txt index b11ea650dc7..f27e9cdbea4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,7 +67,7 @@ if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURC message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive") endif () -include (cmake/find/ccache.cmake) +include (cmake/ccache.cmake) # Take care to add prlimit in command line before ccache, or else ccache thinks that # prlimit is compiler, and clang++ is its input file, and refuses to work with @@ -182,7 +182,7 @@ if (COMPILER_CLANG) if (HAS_USE_CTOR_HOMING) # For more info see https://blog.llvm.org/posts/2021-04-05-constructor-homing-for-debug-info/ - if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO") + if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG" OR CMAKE_BUILD_TYPE_UC STREQUAL "RELWITHDEBINFO") set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Xclang -fuse-ctor-homing") set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -Xclang -fuse-ctor-homing") endif() @@ -247,8 +247,6 @@ endif() if (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG") set(USE_DEBUG_HELPERS ON) -else () - set(USE_DEBUG_HELPERS ON) endif() option(USE_DEBUG_HELPERS "Enable debug helpers" ${USE_DEBUG_HELPERS}) @@ -403,17 +401,6 @@ else () option(WERROR "Enable -Werror compiler option" ON) endif () -if (WERROR) - # Don't pollute CMAKE_CXX_FLAGS with -Werror as it will break some CMake checks. - # Instead, adopt modern cmake usage requirement. - target_compile_options(global-libs INTERFACE "-Werror") -endif () - -# Make this extra-checks for correct library dependencies. -if (OS_LINUX AND NOT SANITIZE) - target_link_options(global-libs INTERFACE "-Wl,--no-undefined") -endif () - # Increase stack size on Musl. We need big stack for our recursive-descend parser. if (USE_MUSL) set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,stack-size=2097152") @@ -421,6 +408,7 @@ endif () include(cmake/dbms_glob_sources.cmake) +add_library(global-group INTERFACE) if (OS_LINUX OR OS_ANDROID) include(cmake/linux/default_libs.cmake) elseif (OS_DARWIN) @@ -428,6 +416,18 @@ elseif (OS_DARWIN) elseif (OS_FREEBSD) include(cmake/freebsd/default_libs.cmake) endif () +link_libraries(global-group) + +if (WERROR) + # Don't pollute CMAKE_CXX_FLAGS with -Werror as it will break some CMake checks. + # Instead, adopt modern cmake usage requirement. + target_compile_options(global-group INTERFACE "-Werror") +endif () + +# Make this extra-checks for correct library dependencies. +if (OS_LINUX AND NOT SANITIZE) + target_link_options(global-group INTERFACE "-Wl,--no-undefined") +endif () ###################################### ### Add targets below this comment ### diff --git a/SECURITY.md b/SECURITY.md index f002dd53ca9..ca3c8b439fd 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -22,9 +22,10 @@ The following versions of ClickHouse server are currently being supported with s | 21.7 | :x: | | 21.8 | ✅ | | 21.9 | :x: | -| 21.10 | ✅ | +| 21.10 | :x: | | 21.11 | ✅ | | 21.12 | ✅ | +| 22.1 | ✅ | ## Reporting a Vulnerability diff --git a/base/base/LineReader.cpp b/base/base/LineReader.cpp index 9491f957762..f4e741a54e7 100644 --- a/base/base/LineReader.cpp +++ b/base/base/LineReader.cpp @@ -2,7 +2,9 @@ #include #include +#include +#include #include #include #include @@ -34,13 +36,37 @@ bool hasInputData() return select(1, &fds, nullptr, nullptr, &timeout) == 1; } +struct NoCaseCompare +{ + bool operator()(const std::string & str1, const std::string & str2) + { + return std::lexicographical_compare(begin(str1), end(str1), begin(str2), end(str2), [](const char c1, const char c2) + { + return std::tolower(c1) < std::tolower(c2); + }); + } +}; + +using Words = std::vector; +template +void addNewWords(Words & to, const Words & from, Compare comp) +{ + size_t old_size = to.size(); + size_t new_size = old_size + from.size(); + + to.reserve(new_size); + to.insert(to.end(), from.begin(), from.end()); + auto middle = to.begin() + old_size; + std::inplace_merge(to.begin(), middle, to.end(), comp); + + auto last_unique = std::unique(to.begin(), to.end()); + to.erase(last_unique, to.end()); } -std::optional LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length) const -{ - if (!ready) - return std::nullopt; +} +replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String & prefix, size_t prefix_length) +{ std::string_view last_word; auto last_word_pos = prefix.find_last_of(word_break_characters); @@ -48,21 +74,45 @@ std::optional LineReader::Suggest::getCompletio last_word = prefix; else last_word = std::string_view(prefix).substr(last_word_pos + 1, std::string::npos); - /// last_word can be empty. + std::pair range; + + std::lock_guard lock(mutex); + /// Only perform case sensitive completion when the prefix string contains any uppercase characters - if (std::none_of(prefix.begin(), prefix.end(), [&](auto c) { return c >= 'A' && c <= 'Z'; })) - return std::equal_range( + if (std::none_of(prefix.begin(), prefix.end(), [](char32_t x) { return iswupper(static_cast(x)); })) + range = std::equal_range( words_no_case.begin(), words_no_case.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched) { return strncasecmp(s.data(), prefix_searched.data(), prefix_length) < 0; }); else - return std::equal_range(words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched) + range = std::equal_range(words.begin(), words.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched) { return strncmp(s.data(), prefix_searched.data(), prefix_length) < 0; }); + + return replxx::Replxx::completions_t(range.first, range.second); +} + +void LineReader::Suggest::addWords(Words && new_words) +{ + Words new_words_no_case = new_words; + if (!new_words.empty()) + { + std::sort(new_words.begin(), new_words.end()); + std::sort(new_words_no_case.begin(), new_words_no_case.end(), NoCaseCompare{}); + } + + { + std::lock_guard lock(mutex); + addNewWords(words, new_words, std::less{}); + addNewWords(words_no_case, new_words_no_case, NoCaseCompare{}); + + assert(std::is_sorted(words.begin(), words.end())); + assert(std::is_sorted(words_no_case.begin(), words_no_case.end(), NoCaseCompare{})); + } } LineReader::LineReader(const String & history_file_path_, bool multiline_, Patterns extenders_, Patterns delimiters_) diff --git a/base/base/LineReader.h b/base/base/LineReader.h index 12a856e2051..33daae49974 100644 --- a/base/base/LineReader.h +++ b/base/base/LineReader.h @@ -1,10 +1,12 @@ #pragma once -#include - +#include #include #include #include +#include + +#include class LineReader { @@ -12,14 +14,16 @@ public: struct Suggest { using Words = std::vector; - using WordsRange = std::pair; + /// Get vector for the matched range of words if any. + replxx::Replxx::completions_t getCompletions(const String & prefix, size_t prefix_length); + void addWords(Words && new_words); + + private: Words words; Words words_no_case; - std::atomic ready{false}; - /// Get iterators for the matched range of words if any. - std::optional getCompletions(const String & prefix, size_t prefix_length) const; + std::mutex mutex; }; using Patterns = std::vector; diff --git a/base/base/ReplxxLineReader.cpp b/base/base/ReplxxLineReader.cpp index 5d99da99c8c..9ea53bb132b 100644 --- a/base/base/ReplxxLineReader.cpp +++ b/base/base/ReplxxLineReader.cpp @@ -25,13 +25,6 @@ void trim(String & s) s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end()); } -/// Check if string ends with given character after skipping whitespaces. -bool ends_with(const std::string_view & s, const std::string_view & p) -{ - auto ss = std::string_view(s.data(), s.rend() - std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); })); - return ss.ends_with(p); -} - std::string getEditor() { const char * editor = std::getenv("EDITOR"); @@ -132,8 +125,14 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx) } +static bool replxx_last_is_delimiter = false; +void ReplxxLineReader::setLastIsDelimiter(bool flag) +{ + replxx_last_is_delimiter = flag; +} + ReplxxLineReader::ReplxxLineReader( - const Suggest & suggest, + Suggest & suggest, const String & history_file_path_, bool multiline_, Patterns extenders_, @@ -179,14 +178,13 @@ ReplxxLineReader::ReplxxLineReader( auto callback = [&suggest] (const String & context, size_t context_size) { - if (auto range = suggest.getCompletions(context, context_size)) - return Replxx::completions_t(range->first, range->second); - return Replxx::completions_t(); + return suggest.getCompletions(context, context_size); }; rx.set_completion_callback(callback); rx.set_complete_on_empty(false); rx.set_word_break_characters(word_break_characters); + rx.set_ignore_case(true); if (highlighter) rx.set_highlighter_callback(highlighter); @@ -198,21 +196,11 @@ ReplxxLineReader::ReplxxLineReader( auto commit_action = [this](char32_t code) { - std::string_view str = rx.get_state().text(); - - /// Always commit line when we see extender at the end. It will start a new prompt. - for (const auto * extender : extenders) - if (ends_with(str, extender)) - return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); - - /// If we see an delimiter at the end, commit right away. - for (const auto * delimiter : delimiters) - if (ends_with(str, delimiter)) - return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); - /// If we allow multiline and there is already something in the input, start a newline. - if (multiline && !input.empty()) + /// NOTE: Lexer is only available if we use highlighter. + if (highlighter && multiline && !replxx_last_is_delimiter) return rx.invoke(Replxx::ACTION::NEW_LINE, code); + replxx_last_is_delimiter = false; return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); }; /// bind C-j to ENTER action. diff --git a/base/base/ReplxxLineReader.h b/base/base/ReplxxLineReader.h index d4cc7de1e7a..b9ec214d02c 100644 --- a/base/base/ReplxxLineReader.h +++ b/base/base/ReplxxLineReader.h @@ -9,7 +9,7 @@ class ReplxxLineReader : public LineReader { public: ReplxxLineReader( - const Suggest & suggest, + Suggest & suggest, const String & history_file_path, bool multiline, Patterns extenders_, @@ -19,6 +19,9 @@ public: void enableBracketedPaste() override; + /// If highlight is on, we will set a flag to denote whether the last token is a delimiter. + /// This is useful to determine the behavior of key when multiline is enabled. + static void setLastIsDelimiter(bool flag); private: InputStatus readOneLine(const String & prompt) override; void addToHistory(const String & line) override; diff --git a/base/base/logger_useful.h b/base/base/logger_useful.h index 1237c6bd47c..ad7d6583f5e 100644 --- a/base/base/logger_useful.h +++ b/base/base/logger_useful.h @@ -12,6 +12,8 @@ namespace { template constexpr size_t numArgs(Ts &&...) { return sizeof...(Ts); } template constexpr auto firstArg(T && x, Ts &&...) { return std::forward(x); } + /// For implicit conversion of fmt::basic_runtime<> to char* for std::string ctor + template constexpr auto firstArg(fmt::basic_runtime && data, Ts &&...) { return data.str.data(); } } diff --git a/base/base/sort.h b/base/base/sort.h index 1f12cc62218..589469fffaa 100644 --- a/base/base/sort.h +++ b/base/base/sort.h @@ -1,26 +1,133 @@ #pragma once +#include + +#ifndef NDEBUG + +#include +#include + +/** Same as libcxx std::__debug_less. Just without dependency on private part of standard library. + * Check that Comparator induce strict weak ordering. + */ +template +class DebugLessComparator +{ +public: + constexpr DebugLessComparator(Comparator & cmp_) + : cmp(cmp_) + {} + + template + constexpr bool operator()(const LhsType & lhs, const RhsType & rhs) + { + bool lhs_less_than_rhs = cmp(lhs, rhs); + if (lhs_less_than_rhs) + assert(!cmp(rhs, lhs)); + + return lhs_less_than_rhs; + } + + template + constexpr bool operator()(LhsType & lhs, RhsType & rhs) + { + bool lhs_less_than_rhs = cmp(lhs, rhs); + if (lhs_less_than_rhs) + assert(!cmp(rhs, lhs)); + + return lhs_less_than_rhs; + } + +private: + Comparator & cmp; +}; + +template +using ComparatorWrapper = DebugLessComparator; + +template +void shuffle(RandomIt first, RandomIt last) +{ + static thread_local pcg64 rng(getThreadId()); + std::shuffle(first, last, rng); +} + +#else + +template +using ComparatorWrapper = Comparator; + +#endif + #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wold-style-cast" #include -template +template void nth_element(RandomIt first, RandomIt nth, RandomIt last) { - ::miniselect::floyd_rivest_select(first, nth, last); + using value_type = typename std::iterator_traits::value_type; + using comparator = std::less; + + comparator compare; + ComparatorWrapper compare_wrapper = compare; + +#ifndef NDEBUG + ::shuffle(first, last); +#endif + + ::miniselect::floyd_rivest_select(first, nth, last, compare_wrapper); + +#ifndef NDEBUG + ::shuffle(first, nth); + + if (nth != last) + ::shuffle(nth + 1, last); +#endif } -template -void partial_sort(RandomIt first, RandomIt middle, RandomIt last) -{ - ::miniselect::floyd_rivest_partial_sort(first, middle, last); -} - -template +template void partial_sort(RandomIt first, RandomIt middle, RandomIt last, Compare compare) { - ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare); +#ifndef NDEBUG + ::shuffle(first, last); +#endif + + ComparatorWrapper compare_wrapper = compare; + ::miniselect::floyd_rivest_partial_sort(first, middle, last, compare_wrapper); + +#ifndef NDEBUG + ::shuffle(middle, last); +#endif +} + +template +void partial_sort(RandomIt first, RandomIt middle, RandomIt last) +{ + using value_type = typename std::iterator_traits::value_type; + using comparator = std::less; + + ::partial_sort(first, middle, last, comparator()); } #pragma GCC diagnostic pop + +template +void sort(RandomIt first, RandomIt last, Compare compare) +{ +#ifndef NDEBUG + ::shuffle(first, last); +#endif + + ComparatorWrapper compare_wrapper = compare; + ::pdqsort(first, last, compare_wrapper); +} + +template +void sort(RandomIt first, RandomIt last) +{ + using value_type = typename std::iterator_traits::value_type; + using comparator = std::less; + ::sort(first, last, comparator()); +} diff --git a/base/base/wide_integer_impl.h b/base/base/wide_integer_impl.h index ec146fd5821..ed2c2972cfe 100644 --- a/base/base/wide_integer_impl.h +++ b/base/base/wide_integer_impl.h @@ -12,6 +12,18 @@ #include #include +#include +#include + +/// Use same extended double for all platforms +#if (LDBL_MANT_DIG == 64) +#define CONSTEXPR_FROM_DOUBLE constexpr +using FromDoubleIntermediateType = long double; +#else +/// `wide_integer_from_builtin` can't be constexpr with non-literal `cpp_bin_float_double_extended` +#define CONSTEXPR_FROM_DOUBLE +using FromDoubleIntermediateType = boost::multiprecision::cpp_bin_float_double_extended; +#endif namespace wide { @@ -265,12 +277,23 @@ struct integer::_impl constexpr static void set_multiplier(integer & self, T t) noexcept { constexpr uint64_t max_int = std::numeric_limits::max(); - + static_assert(std::is_same_v || std::is_same_v); /// Implementation specific behaviour on overflow (if we don't check here, stack overflow will triggered in bigint_cast). - if (!std::isfinite(t)) + if constexpr (std::is_same_v) { - self = 0; - return; + if (!std::isfinite(t)) + { + self = 0; + return; + } + } + else + { + if (!boost::math::isfinite(t)) + { + self = 0; + return; + } } const T alpha = t / static_cast(max_int); @@ -278,13 +301,13 @@ struct integer::_impl if (alpha <= static_cast(max_int)) self = static_cast(alpha); else // max(double) / 2^64 will surely contain less than 52 precision bits, so speed up computations. - set_multiplier(self, alpha); + set_multiplier(self, static_cast(alpha)); self *= max_int; self += static_cast(t - floor(alpha) * static_cast(max_int)); // += b_i } - constexpr static void wide_integer_from_builtin(integer & self, double rhs) noexcept + CONSTEXPR_FROM_DOUBLE static void wide_integer_from_builtin(integer & self, double rhs) noexcept { constexpr int64_t max_int = std::numeric_limits::max(); constexpr int64_t min_int = std::numeric_limits::lowest(); @@ -294,24 +317,17 @@ struct integer::_impl /// the result may not fit in 64 bits. /// The example of such a number is 9.22337e+18. /// As to_Integral does a static_cast to int64_t, it may result in UB. - /// The necessary check here is that long double has enough significant (mantissa) bits to store the + /// The necessary check here is that FromDoubleIntermediateType has enough significant (mantissa) bits to store the /// int64_t max value precisely. - // TODO Be compatible with Apple aarch64 -#if not (defined(__APPLE__) && defined(__aarch64__)) - static_assert(LDBL_MANT_DIG >= 64, - "On your system long double has less than 64 precision bits, " - "which may result in UB when initializing double from int64_t"); -#endif - - if (rhs > static_cast(min_int) && rhs < static_cast(max_int)) + if (rhs > static_cast(min_int) && rhs < static_cast(max_int)) { self = static_cast(rhs); return; } - const long double rhs_long_double = (static_cast(rhs) < 0) - ? -static_cast(rhs) + const FromDoubleIntermediateType rhs_long_double = (static_cast(rhs) < 0) + ? -static_cast(rhs) : rhs; set_multiplier(self, rhs_long_double); diff --git a/base/daemon/BaseDaemon.cpp b/base/daemon/BaseDaemon.cpp index f3026d7c87a..311349a2ba7 100644 --- a/base/daemon/BaseDaemon.cpp +++ b/base/daemon/BaseDaemon.cpp @@ -79,18 +79,14 @@ static void call_default_signal_handler(int sig) raise(sig); } -static constexpr size_t max_query_id_size = 127; - static const size_t signal_pipe_buf_size = sizeof(int) + sizeof(siginfo_t) - + sizeof(ucontext_t) + + sizeof(ucontext_t*) + sizeof(StackTrace) + sizeof(UInt32) - + max_query_id_size + 1 /// query_id + varint encoded length + sizeof(void*); - using signal_function = void(int, siginfo_t*, void*); static void writeSignalIDtoSignalPipe(int sig) @@ -129,18 +125,14 @@ static void signalHandler(int sig, siginfo_t * info, void * context) char buf[signal_pipe_buf_size]; DB::WriteBufferFromFileDescriptorDiscardOnFailure out(signal_pipe.fds_rw[1], signal_pipe_buf_size, buf); - const ucontext_t signal_context = *reinterpret_cast(context); - const StackTrace stack_trace(signal_context); - - StringRef query_id = DB::CurrentThread::getQueryId(); /// This is signal safe. - query_id.size = std::min(query_id.size, max_query_id_size); + const ucontext_t * signal_context = reinterpret_cast(context); + const StackTrace stack_trace(*signal_context); DB::writeBinary(sig, out); DB::writePODBinary(*info, out); DB::writePODBinary(signal_context, out); DB::writePODBinary(stack_trace, out); DB::writeBinary(UInt32(getThreadId()), out); - DB::writeStringBinary(query_id, out); DB::writePODBinary(DB::current_thread, out); out.next(); @@ -184,6 +176,8 @@ public: void run() override { + static_assert(PIPE_BUF >= 512); + static_assert(signal_pipe_buf_size <= PIPE_BUF, "Only write of PIPE_BUF to pipe is atomic and the minimal known PIPE_BUF across supported platforms is 512"); char buf[signal_pipe_buf_size]; DB::ReadBufferFromFileDescriptor in(signal_pipe.fds_rw[0], signal_pipe_buf_size, buf); @@ -227,10 +221,9 @@ public: else { siginfo_t info{}; - ucontext_t context{}; + ucontext_t * context{}; StackTrace stack_trace(NoCapture{}); UInt32 thread_num{}; - std::string query_id; DB::ThreadStatus * thread_ptr{}; if (sig != SanitizerTrap) @@ -241,12 +234,11 @@ public: DB::readPODBinary(stack_trace, in); DB::readBinary(thread_num, in); - DB::readBinary(query_id, in); DB::readPODBinary(thread_ptr, in); /// This allows to receive more signals if failure happens inside onFault function. /// Example: segfault while symbolizing stack trace. - std::thread([=, this] { onFault(sig, info, context, stack_trace, thread_num, query_id, thread_ptr); }).detach(); + std::thread([=, this] { onFault(sig, info, context, stack_trace, thread_num, thread_ptr); }).detach(); } } } @@ -279,18 +271,27 @@ private: void onFault( int sig, const siginfo_t & info, - const ucontext_t & context, + ucontext_t * context, const StackTrace & stack_trace, UInt32 thread_num, - const std::string & query_id, DB::ThreadStatus * thread_ptr) const { DB::ThreadStatus thread_status; + String query_id; + String query; + /// Send logs from this thread to client if possible. /// It will allow client to see failure messages directly. if (thread_ptr) { + query_id = thread_ptr->getQueryId().toString(); + + if (auto thread_group = thread_ptr->getThreadGroup()) + { + query = thread_group->query; + } + if (auto logs_queue = thread_ptr->getInternalTextLogsQueue()) DB::CurrentThread::attachInternalTextLogsQueue(logs_queue, DB::LogsLevel::trace); } @@ -305,19 +306,19 @@ private: } else { - LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) Received signal {} ({})", + LOG_FATAL(log, "(version {}{}, {}) (from thread {}) (query_id: {}) (query: {}) Received signal {} ({})", VERSION_STRING, VERSION_OFFICIAL, daemon.build_id_info, - thread_num, query_id, strsignal(sig), sig); + thread_num, query_id, query, strsignal(sig), sig); } String error_message; if (sig != SanitizerTrap) - error_message = signalToErrorMessage(sig, info, context); + error_message = signalToErrorMessage(sig, info, *context); else error_message = "Sanitizer trap."; - LOG_FATAL(log, error_message); + LOG_FATAL(log, fmt::runtime(error_message)); if (stack_trace.getSize()) { @@ -330,11 +331,11 @@ private: for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i) bare_stacktrace << ' ' << stack_trace.getFramePointers()[i]; - LOG_FATAL(log, bare_stacktrace.str()); + LOG_FATAL(log, fmt::runtime(bare_stacktrace.str())); } /// Write symbolized stack trace line by line for better grep-ability. - stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); }); + stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, fmt::runtime(s)); }); #if defined(OS_LINUX) /// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace. @@ -389,20 +390,16 @@ static void sanitizerDeathCallback() const StackTrace stack_trace; - StringRef query_id = DB::CurrentThread::getQueryId(); - query_id.size = std::min(query_id.size, max_query_id_size); - int sig = SignalListener::SanitizerTrap; DB::writeBinary(sig, out); DB::writePODBinary(stack_trace, out); DB::writeBinary(UInt32(getThreadId()), out); - DB::writeStringBinary(query_id, out); DB::writePODBinary(DB::current_thread, out); out.next(); /// The time that is usually enough for separate thread to print info into log. - sleepForSeconds(10); + sleepForSeconds(20); } #endif diff --git a/cmake/find/ccache.cmake b/cmake/ccache.cmake similarity index 100% rename from cmake/find/ccache.cmake rename to cmake/ccache.cmake diff --git a/cmake/find/cxx.cmake b/cmake/cxx.cmake similarity index 100% rename from cmake/find/cxx.cmake rename to cmake/cxx.cmake diff --git a/cmake/darwin/default_libs.cmake b/cmake/darwin/default_libs.cmake index a6ee800d59b..1f92663a4b9 100644 --- a/cmake/darwin/default_libs.cmake +++ b/cmake/darwin/default_libs.cmake @@ -22,16 +22,12 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -include (cmake/find/cxx.cmake) - -add_library(global-group INTERFACE) +include (cmake/cxx.cmake) target_link_libraries(global-group INTERFACE $ ) -link_libraries(global-group) - # FIXME: remove when all contribs will get custom cmake lists install( TARGETS global-group global-libs diff --git a/cmake/freebsd/default_libs.cmake b/cmake/freebsd/default_libs.cmake index a5847c95387..65d5f0511d9 100644 --- a/cmake/freebsd/default_libs.cmake +++ b/cmake/freebsd/default_libs.cmake @@ -22,17 +22,13 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS}) set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) -include (cmake/find/unwind.cmake) -include (cmake/find/cxx.cmake) - -add_library(global-group INTERFACE) +include (cmake/unwind.cmake) +include (cmake/cxx.cmake) target_link_libraries(global-group INTERFACE $ ) -link_libraries(global-group) - # FIXME: remove when all contribs will get custom cmake lists install( TARGETS global-group global-libs diff --git a/cmake/linux/default_libs.cmake b/cmake/linux/default_libs.cmake index 426ae482ea3..21bead7020c 100644 --- a/cmake/linux/default_libs.cmake +++ b/cmake/linux/default_libs.cmake @@ -42,18 +42,15 @@ if (NOT OS_ANDROID) add_subdirectory(base/harmful) endif () -include (cmake/find/unwind.cmake) -include (cmake/find/cxx.cmake) +include (cmake/unwind.cmake) +include (cmake/cxx.cmake) -add_library(global-group INTERFACE) target_link_libraries(global-group INTERFACE -Wl,--start-group $ -Wl,--end-group ) -link_libraries(global-group) - # FIXME: remove when all contribs will get custom cmake lists install( TARGETS global-group global-libs diff --git a/cmake/find/unwind.cmake b/cmake/unwind.cmake similarity index 100% rename from cmake/find/unwind.cmake rename to cmake/unwind.cmake diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 6172f231b6e..9cf307c473e 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -78,6 +78,7 @@ add_contrib (croaring-cmake croaring) add_contrib (zstd-cmake zstd) add_contrib (zlib-ng-cmake zlib-ng) add_contrib (bzip2-cmake bzip2) +add_contrib (minizip-ng-cmake minizip-ng) add_contrib (snappy-cmake snappy) add_contrib (rocksdb-cmake rocksdb) add_contrib (thrift-cmake thrift) diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 54bfead6da7..2e4059efc17 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -29,12 +29,6 @@ if (OS_FREEBSD) message (FATAL_ERROR "Using internal parquet library on FreeBSD is not supported") endif() -if(USE_STATIC_LIBRARIES) - set(FLATBUFFERS_LIBRARY flatbuffers) -else() - set(FLATBUFFERS_LIBRARY flatbuffers_shared) -endif() - set (CMAKE_CXX_STANDARD 17) set(ARROW_VERSION "6.0.1") @@ -95,9 +89,16 @@ set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Skip flatbuffers tests") add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}") -message(STATUS "FLATBUFFERS_LIBRARY: ${FLATBUFFERS_LIBRARY}") +add_library(_flatbuffers INTERFACE) +if(USE_STATIC_LIBRARIES) + target_link_libraries(_flatbuffers INTERFACE flatbuffers) +else() + target_link_libraries(_flatbuffers INTERFACE flatbuffers_shared) +endif() +target_include_directories(_flatbuffers INTERFACE ${FLATBUFFERS_INCLUDE_DIR}) # === hdfs +# NOTE: cannot use ch_contrib::hdfs since it's INCLUDE_DIRECTORIES does not includes trailing "hdfs/" set(HDFS_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/") # arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features. @@ -123,8 +124,6 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A # ARROW_ORC + adapters/orc/CMakefiles set(ORC_SRCS - "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc" - "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc" "${ORC_SOURCE_SRC_DIR}/Exceptions.cc" "${ORC_SOURCE_SRC_DIR}/OrcFile.cc" "${ORC_SOURCE_SRC_DIR}/Reader.cc" @@ -151,6 +150,22 @@ set(ORC_SRCS "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc" ) +add_library(_orc ${ORC_SRCS}) +target_link_libraries(_orc PRIVATE + ch_contrib::protobuf + ch_contrib::lz4 + ch_contrib::snappy + ch_contrib::zlib + ch_contrib::zstd) +target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR}) +target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) +target_include_directories(_orc SYSTEM PRIVATE + ${ORC_SOURCE_SRC_DIR} + ${ORC_SOURCE_WRAP_DIR} + ${ORC_BUILD_SRC_DIR} + ${ORC_ADDITION_SOURCE_DIR} + ${ARROW_SRC_DIR}) + # === arrow @@ -336,7 +351,8 @@ set(ARROW_SRCS "${LIBRARY_DIR}/ipc/reader.cc" "${LIBRARY_DIR}/ipc/writer.cc" - ${ORC_SRCS} + "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc" + "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc" ) add_definitions(-DARROW_WITH_LZ4) @@ -356,30 +372,27 @@ endif () add_library(_arrow ${ARROW_SRCS}) -# Arrow dependencies -add_dependencies(_arrow ${FLATBUFFERS_LIBRARY}) +target_link_libraries(_arrow PRIVATE + boost::filesystem -target_link_libraries(_arrow PRIVATE ${FLATBUFFERS_LIBRARY} boost::filesystem) + _flatbuffers + + ch_contrib::double_conversion + + ch_contrib::lz4 + ch_contrib::snappy + ch_contrib::zlib + ch_contrib::zstd + ch_contrib::zstd +) +target_link_libraries(_arrow PUBLIC _orc) add_dependencies(_arrow protoc) target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src") -target_link_libraries(_arrow PRIVATE ch_contrib::double_conversion) -target_link_libraries(_arrow PRIVATE ch_contrib::protobuf) -target_link_libraries(_arrow PRIVATE ch_contrib::lz4) -target_link_libraries(_arrow PRIVATE ch_contrib::snappy) -target_link_libraries(_arrow PRIVATE ch_contrib::zlib) -target_link_libraries(_arrow PRIVATE ch_contrib::zstd) -target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR}) -target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_SRC_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_WRAP_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_BUILD_SRC_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${ORC_ADDITION_SOURCE_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${ARROW_SRC_DIR}) -target_include_directories(_arrow SYSTEM PRIVATE ${FLATBUFFERS_INCLUDE_DIR}) target_include_directories(_arrow SYSTEM PRIVATE ${HDFS_INCLUDE_DIR}) # === parquet diff --git a/contrib/cassandra-cmake/CMakeLists.txt b/contrib/cassandra-cmake/CMakeLists.txt index 416dca6f2bc..81c1fab3882 100644 --- a/contrib/cassandra-cmake/CMakeLists.txt +++ b/contrib/cassandra-cmake/CMakeLists.txt @@ -56,19 +56,11 @@ list(APPEND SOURCES ${CASS_SRC_DIR}/atomic/atomic_std.hpp) add_library(_curl_hostcheck OBJECT ${CASS_SRC_DIR}/third_party/curl/hostcheck.cpp) add_library(_hdr_histogram OBJECT ${CASS_SRC_DIR}/third_party/hdr_histogram/hdr_histogram.cpp) add_library(_http-parser OBJECT ${CASS_SRC_DIR}/third_party/http-parser/http_parser.c) -add_library(_minizip OBJECT - ${CASS_SRC_DIR}/third_party/minizip/ioapi.c - ${CASS_SRC_DIR}/third_party/minizip/zip.c - ${CASS_SRC_DIR}/third_party/minizip/unzip.c) - -target_link_libraries(_minizip ch_contrib::zlib) -target_compile_definitions(_minizip PRIVATE "-Dz_crc_t=unsigned long") list(APPEND INCLUDE_DIRS ${CASS_SRC_DIR}/third_party/curl ${CASS_SRC_DIR}/third_party/hdr_histogram ${CASS_SRC_DIR}/third_party/http-parser - ${CASS_SRC_DIR}/third_party/minizip ${CASS_SRC_DIR}/third_party/mt19937_64 ${CASS_SRC_DIR}/third_party/rapidjson/rapidjson ${CASS_SRC_DIR}/third_party/sparsehash/src) @@ -123,10 +115,9 @@ add_library(_cassandra ${SOURCES} $ $ - $ - $) + $) -target_link_libraries(_cassandra ch_contrib::zlib) +target_link_libraries(_cassandra ch_contrib::zlib ch_contrib::minizip) target_include_directories(_cassandra PRIVATE ${CMAKE_CURRENT_BINARY_DIR} ${INCLUDE_DIRS}) target_include_directories(_cassandra SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) target_compile_definitions(_cassandra PRIVATE CASS_BUILDING) diff --git a/contrib/fmtlib b/contrib/fmtlib index c108ee1d590..b6f4ceaed0a 160000 --- a/contrib/fmtlib +++ b/contrib/fmtlib @@ -1 +1 @@ -Subproject commit c108ee1d590089ccf642fc85652b845924067af2 +Subproject commit b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9 diff --git a/contrib/fmtlib-cmake/CMakeLists.txt b/contrib/fmtlib-cmake/CMakeLists.txt index d8cb721b9ba..fecec5f3e43 100644 --- a/contrib/fmtlib-cmake/CMakeLists.txt +++ b/contrib/fmtlib-cmake/CMakeLists.txt @@ -1,7 +1,10 @@ set (SRCS + # NOTE: do not build module for now: + # ../fmtlib/src/fmt.cc ../fmtlib/src/format.cc ../fmtlib/src/os.cc + ../fmtlib/include/fmt/args.h ../fmtlib/include/fmt/chrono.h ../fmtlib/include/fmt/color.h ../fmtlib/include/fmt/compile.h @@ -11,9 +14,9 @@ set (SRCS ../fmtlib/include/fmt/locale.h ../fmtlib/include/fmt/os.h ../fmtlib/include/fmt/ostream.h - ../fmtlib/include/fmt/posix.h ../fmtlib/include/fmt/printf.h ../fmtlib/include/fmt/ranges.h + ../fmtlib/include/fmt/xchar.h ) add_library(_fmt ${SRCS}) diff --git a/contrib/minizip-ng b/contrib/minizip-ng new file mode 160000 index 00000000000..6cffc951851 --- /dev/null +++ b/contrib/minizip-ng @@ -0,0 +1 @@ +Subproject commit 6cffc951851620e0fac1993be75e4713c334de03 diff --git a/contrib/minizip-ng-cmake/CMakeLists.txt b/contrib/minizip-ng-cmake/CMakeLists.txt new file mode 100644 index 00000000000..4aabbd3c9fb --- /dev/null +++ b/contrib/minizip-ng-cmake/CMakeLists.txt @@ -0,0 +1,168 @@ +option(ENABLE_MINIZIP "Enable minizip-ng the zip manipulation library" ${ENABLE_LIBRARIES}) +if (NOT ENABLE_MINIZIP) + message (STATUS "minizip-ng disabled") + return() +endif() + +set(_MINIZIP_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/minizip-ng") + +# Initial source files +set(MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_crypt.c + ${_MINIZIP_SOURCE_DIR}/mz_os.c + ${_MINIZIP_SOURCE_DIR}/mz_strm.c + ${_MINIZIP_SOURCE_DIR}/mz_strm_buf.c + ${_MINIZIP_SOURCE_DIR}/mz_strm_mem.c + ${_MINIZIP_SOURCE_DIR}/mz_strm_split.c + ${_MINIZIP_SOURCE_DIR}/mz_zip.c + ${_MINIZIP_SOURCE_DIR}/mz_zip_rw.c) + +# Initial header files +set(MINIZIP_HDR + ${_MINIZIP_SOURCE_DIR}/mz.h + ${_MINIZIP_SOURCE_DIR}/mz_os.h + ${_MINIZIP_SOURCE_DIR}/mz_crypt.h + ${_MINIZIP_SOURCE_DIR}/mz_strm.h + ${_MINIZIP_SOURCE_DIR}/mz_strm_buf.h + ${_MINIZIP_SOURCE_DIR}/mz_strm_mem.h + ${_MINIZIP_SOURCE_DIR}/mz_strm_split.h + ${_MINIZIP_SOURCE_DIR}/mz_strm_os.h + ${_MINIZIP_SOURCE_DIR}/mz_zip.h + ${_MINIZIP_SOURCE_DIR}/mz_zip_rw.h) + +set(MINIZIP_INC ${_MINIZIP_SOURCE_DIR}) + +set(MINIZIP_DEF) +set(MINIZIP_PUBLIC_DEF) +set(MINIZIP_LIB) + +# Check if zlib is present +set(MZ_ZLIB ON) +if(MZ_ZLIB) + # Use zlib from ClickHouse contrib + list(APPEND MINIZIP_LIB ch_contrib::zlib) + + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_strm_zlib.c) + + list(APPEND MINIZIP_HDR + ${_MINIZIP_SOURCE_DIR}/mz_strm_zlib.h) + + list(APPEND MINIZIP_DEF "-DHAVE_ZLIB") +endif() + +# Check if bzip2 is present +set(MZ_BZIP2 ${ENABLE_BZIP2}) +if(MZ_BZIP2) + # Use bzip2 from ClickHouse contrib + list(APPEND MINIZIP_LIB ch_contrib::bzip2) + + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_strm_bzip.c) + + list(APPEND MINIZIP_HDR + ${_MINIZIP_SOURCE_DIR}/mz_strm_bzip.h) + + list(APPEND MINIZIP_DEF "-DHAVE_BZIP2") +endif() + +# Check if liblzma is present +set(MZ_LZMA ON) +if(MZ_LZMA) + # Use liblzma from ClickHouse contrib + list(APPEND MINIZIP_LIB ch_contrib::xz) + + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_strm_lzma.c) + + list(APPEND MINIZIP_HDR + ${_MINIZIP_SOURCE_DIR}/mz_strm_lzma.h) + + list(APPEND MINIZIP_DEF "-DHAVE_LZMA") +endif() + +# Check if zstd is present +set(MZ_ZSTD ON) +if(MZ_ZSTD) + # Use zstd from ClickHouse contrib + list(APPEND MINIZIP_LIB ch_contrib::zstd) + + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_strm_zstd.c) + + list(APPEND MINIZIP_HDR + ${_MINIZIP_SOURCE_DIR}/mz_strm_zstd.h) + + list(APPEND MINIZIP_DEF "-DHAVE_ZSTD") +endif() + +if(NOT MZ_ZLIB AND NOT MZ_ZSTD AND NOT MZ_BZIP2 AND NOT MZ_LZMA) + message(STATUS "Compression not supported due to missing libraries") + + list(APPEND MINIZIP_DEF -DMZ_ZIP_NO_DECOMPRESSION) + list(APPEND MINIZIP_DEF -DMZ_ZIP_NO_COMPRESSION) +endif() + +# Check to see if openssl installation is present +set(MZ_OPENSSL ${ENABLE_SSL}) +if(MZ_OPENSSL) + # Use openssl from ClickHouse contrib + list(APPEND MINIZIP_LIB OpenSSL::SSL OpenSSL::Crypto) + + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_crypt_openssl.c) +endif() + +# Include WinZIP AES encryption +set(MZ_WZAES ${ENABLE_SSL}) +if(MZ_WZAES) + list(APPEND MINIZIP_DEF -DHAVE_WZAES) + + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_strm_wzaes.c) + + list(APPEND MINIZIP_HDR + ${_MINIZIP_SOURCE_DIR}/mz_strm_wzaes.h) +endif() + +# Include traditional PKWare encryption +set(MZ_PKCRYPT ON) +if(MZ_PKCRYPT) + list(APPEND MINIZIP_DEF -DHAVE_PKCRYPT) + + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_strm_pkcrypt.c) + + list(APPEND MINIZIP_HDR + ${_MINIZIP_SOURCE_DIR}/mz_strm_pkcrypt.h) +endif() + +# Unix specific +if(UNIX) + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_os_posix.c + ${_MINIZIP_SOURCE_DIR}/mz_strm_os_posix.c) +endif() + +# Include compatibility layer +set(MZ_COMPAT ON) +if(MZ_COMPAT) + list(APPEND MINIZIP_SRC + ${_MINIZIP_SOURCE_DIR}/mz_compat.c) + + list(APPEND MINIZIP_HDR + ${_MINIZIP_SOURCE_DIR}/mz_compat.h + zip.h + unzip.h) + + list(APPEND MINIZIP_INC "${CMAKE_CURRENT_SOURCE_DIR}") + list(APPEND MINIZIP_PUBLIC_DEF "-DMZ_COMPAT_VERSION=110") +endif() + +add_library(_minizip ${MINIZIP_SRC} ${MINIZIP_HDR}) +target_include_directories(_minizip PUBLIC ${MINIZIP_INC}) +target_compile_definitions(_minizip PUBLIC ${MINIZIP_PUBLIC_DEF}) +target_compile_definitions(_minizip PRIVATE ${MINIZIP_DEF}) +target_link_libraries(_minizip PRIVATE ${MINIZIP_LIB}) + +add_library(ch_contrib::minizip ALIAS _minizip) diff --git a/contrib/minizip-ng-cmake/unzip.h b/contrib/minizip-ng-cmake/unzip.h new file mode 100644 index 00000000000..61cbd974e31 --- /dev/null +++ b/contrib/minizip-ng-cmake/unzip.h @@ -0,0 +1,13 @@ +/* unzip.h -- Compatibility layer shim + part of the minizip-ng project + + This program is distributed under the terms of the same license as zlib. + See the accompanying LICENSE file for the full text of the license. +*/ + +#ifndef MZ_COMPAT_UNZIP +#define MZ_COMPAT_UNZIP + +#include "mz_compat.h" + +#endif diff --git a/contrib/minizip-ng-cmake/zip.h b/contrib/minizip-ng-cmake/zip.h new file mode 100644 index 00000000000..cf38ac91a04 --- /dev/null +++ b/contrib/minizip-ng-cmake/zip.h @@ -0,0 +1,13 @@ +/* zip.h -- Compatibility layer shim + part of the minizip-ng project + + This program is distributed under the terms of the same license as zlib. + See the accompanying LICENSE file for the full text of the license. +*/ + +#ifndef MZ_COMPAT_ZIP +#define MZ_COMPAT_ZIP + +#include "mz_compat.h" + +#endif diff --git a/contrib/orc b/contrib/orc index 0a936f6bbdb..f9a393ed243 160000 --- a/contrib/orc +++ b/contrib/orc @@ -1 +1 @@ -Subproject commit 0a936f6bbdb9303308973073f8623b5a8d82eae1 +Subproject commit f9a393ed2433a60034795284f82d093b348f2102 diff --git a/contrib/replxx b/contrib/replxx index f019cba7ea1..9460e5e0fc1 160000 --- a/contrib/replxx +++ b/contrib/replxx @@ -1 +1 @@ -Subproject commit f019cba7ea1bcd1b4feb7826f28ed57fb581b04c +Subproject commit 9460e5e0fc10f78f460af26a6bd928798cac864d diff --git a/contrib/rocksdb-cmake/CMakeLists.txt b/contrib/rocksdb-cmake/CMakeLists.txt index 902d29a9630..529d7f0c4e3 100644 --- a/contrib/rocksdb-cmake/CMakeLists.txt +++ b/contrib/rocksdb-cmake/CMakeLists.txt @@ -72,11 +72,6 @@ else() if(WITH_ZSTD) add_definitions(-DZSTD) - include_directories(${ZSTD_INCLUDE_DIR}) - include_directories("${ZSTD_INCLUDE_DIR}/common") - include_directories("${ZSTD_INCLUDE_DIR}/dictBuilder") - include_directories("${ZSTD_INCLUDE_DIR}/deprecated") - list(APPEND THIRDPARTY_LIBS ch_contrib::zstd) endif() endif() @@ -132,11 +127,6 @@ endif() if(CMAKE_SYSTEM_NAME MATCHES "Darwin") add_definitions(-DOS_MACOSX) - if(CMAKE_SYSTEM_PROCESSOR MATCHES arm) - add_definitions(-DIOS_CROSS_COMPILE -DROCKSDB_LITE) - # no debug info for IOS, that will make our library big - add_definitions(-DNDEBUG) - endif() elseif(CMAKE_SYSTEM_NAME MATCHES "Linux") add_definitions(-DOS_LINUX) elseif(CMAKE_SYSTEM_NAME MATCHES "SunOS") diff --git a/debian/clickhouse-server.service b/debian/clickhouse-server.service index bc19235cb3a..a9400b24270 100644 --- a/debian/clickhouse-server.service +++ b/debian/clickhouse-server.service @@ -16,6 +16,8 @@ Restart=always RestartSec=30 RuntimeDirectory=clickhouse-server ExecStart=/usr/bin/clickhouse-server --config=/etc/clickhouse-server/config.xml --pid-file=/run/clickhouse-server/clickhouse-server.pid +# Minus means that this file is optional. +EnvironmentFile=-/etc/default/clickhouse LimitCORE=infinity LimitNOFILE=500000 CapabilityBoundingSet=CAP_NET_ADMIN CAP_IPC_LOCK CAP_SYS_NICE diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 3e3cfc38218..00000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,15 +0,0 @@ -version: "2" - -services: - builder: - image: clickhouse/clickhouse-builder - build: docker/builder - client: - image: clickhouse/clickhouse-client - build: docker/client - command: ['--host', 'server'] - server: - image: clickhouse/clickhouse-server - build: docker/server - ports: - - 8123:8123 diff --git a/docker/images.json b/docker/images.json index 354bdaa8728..01284d4de69 100644 --- a/docker/images.json +++ b/docker/images.json @@ -32,6 +32,7 @@ "dependent": [] }, "docker/test/pvs": { + "only_amd64": true, "name": "clickhouse/pvs-test", "dependent": [] }, @@ -72,6 +73,7 @@ "dependent": [] }, "docker/test/integration/runner": { + "only_amd64": true, "name": "clickhouse/integration-tests-runner", "dependent": [] }, @@ -124,6 +126,7 @@ "dependent": [] }, "docker/test/integration/kerberos_kdc": { + "only_amd64": true, "name": "clickhouse/kerberos-kdc", "dependent": [] }, @@ -137,6 +140,7 @@ ] }, "docker/test/integration/kerberized_hadoop": { + "only_amd64": true, "name": "clickhouse/kerberized-hadoop", "dependent": [] }, diff --git a/docker/test/fuzzer/run-fuzzer.sh b/docker/test/fuzzer/run-fuzzer.sh index 1ebaed752a6..e18c07bf2c1 100755 --- a/docker/test/fuzzer/run-fuzzer.sh +++ b/docker/test/fuzzer/run-fuzzer.sh @@ -185,15 +185,14 @@ handle SIGUSR2 nostop noprint pass handle SIG$RTMIN nostop noprint pass info signals continue +gcore backtrace full -info locals +thread apply all backtrace full info registers disassemble /s up -info locals disassemble /s up -info locals disassemble /s p \"done\" detach @@ -314,6 +313,11 @@ quit || echo "Fuzzer failed ($fuzzer_exit_code). See the logs." ; } \ | tail -1 > description.txt fi + + if test -f core.*; then + pigz core.* + mv core.*.gz core.gz + fi } case "$stage" in @@ -345,6 +349,10 @@ case "$stage" in time fuzz ;& "report") +CORE_LINK='' +if [ -f core.gz ]; then + CORE_LINK='core.gz' +fi cat > report.html < @@ -386,6 +394,7 @@ th { cursor: pointer; } fuzzer.log server.log main.log +${CORE_LINK}

diff --git a/docker/test/integration/kerberized_hadoop/Dockerfile b/docker/test/integration/kerberized_hadoop/Dockerfile index 025f4b27fde..592c3e36ef7 100644 --- a/docker/test/integration/kerberized_hadoop/Dockerfile +++ b/docker/test/integration/kerberized_hadoop/Dockerfile @@ -15,9 +15,10 @@ RUN curl -o krb5-libs-1.10.3-65.el6.x86_64.rpm ftp://ftp.pbone.net/mirror/vault. rm -fr *.rpm RUN cd /tmp && \ - curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \ - tar xzf commons-daemon-1.0.15-src.tar.gz && \ - cd commons-daemon-1.0.15-src/src/native/unix && \ - ./configure && \ - make && \ - cp ./jsvc /usr/local/hadoop/sbin + curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \ + tar xzf commons-daemon-1.0.15-src.tar.gz && \ + cd commons-daemon-1.0.15-src/src/native/unix && \ + ./configure && \ + make && \ + cp ./jsvc /usr/local/hadoop-2.7.0/sbin && \ + [ -e /usr/local/hadoop ] || ln -s ./hadoop-2.7.0 /usr/local/hadoop diff --git a/docker/test/integration/runner/Dockerfile b/docker/test/integration/runner/Dockerfile index 1aad2ae6770..22dd2e14456 100644 --- a/docker/test/integration/runner/Dockerfile +++ b/docker/test/integration/runner/Dockerfile @@ -58,9 +58,7 @@ RUN apt-get update \ RUN dockerd --version; docker --version -ARG TARGETARCH -# FIXME: psycopg2-binary is not available for aarch64, we skip it for now -RUN test x$TARGETARCH = xarm64 || ( python3 -m pip install \ +RUN python3 -m pip install \ PyMySQL \ aerospike==4.0.0 \ avro==1.10.2 \ @@ -90,7 +88,7 @@ RUN test x$TARGETARCH = xarm64 || ( python3 -m pip install \ urllib3 \ requests-kerberos \ pyhdfs \ - azure-storage-blob ) + azure-storage-blob COPY modprobe.sh /usr/local/bin/modprobe COPY dockerd-entrypoint.sh /usr/local/bin/ diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml index 88be3e45085..e1b4d393169 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml @@ -4,7 +4,7 @@ services: kerberizedhdfs1: cap_add: - DAC_READ_SEARCH - image: clickhouse/kerberized-hadoop + image: clickhouse/kerberized-hadoop:${DOCKER_KERBERIZED_HADOOP_TAG:-latest} hostname: kerberizedhdfs1 restart: always volumes: diff --git a/docker/test/integration/runner/dockerd-entrypoint.sh b/docker/test/integration/runner/dockerd-entrypoint.sh index 8109ef7ae64..34414abc3f5 100755 --- a/docker/test/integration/runner/dockerd-entrypoint.sh +++ b/docker/test/integration/runner/dockerd-entrypoint.sh @@ -45,6 +45,7 @@ export DOCKER_MYSQL_JS_CLIENT_TAG=${DOCKER_MYSQL_JS_CLIENT_TAG:=latest} export DOCKER_MYSQL_PHP_CLIENT_TAG=${DOCKER_MYSQL_PHP_CLIENT_TAG:=latest} export DOCKER_POSTGRESQL_JAVA_CLIENT_TAG=${DOCKER_POSTGRESQL_JAVA_CLIENT_TAG:=latest} export DOCKER_KERBEROS_KDC_TAG=${DOCKER_KERBEROS_KDC_TAG:=latest} +export DOCKER_KERBERIZED_HADOOP_TAG=${DOCKER_KERBERIZED_HADOOP_TAG:=latest} cd /ClickHouse/tests/integration exec "$@" diff --git a/docker/test/performance-comparison/Dockerfile b/docker/test/performance-comparison/Dockerfile index eddaf969f33..fb47ed0cefa 100644 --- a/docker/test/performance-comparison/Dockerfile +++ b/docker/test/performance-comparison/Dockerfile @@ -1,5 +1,5 @@ # docker build -t clickhouse/performance-comparison . -FROM ubuntu:18.04 +FROM ubuntu:20.04 # ARG for quick switch to a given ubuntu mirror ARG apt_archive="http://archive.ubuntu.com" diff --git a/docker/test/pvs/Dockerfile b/docker/test/pvs/Dockerfile index f484feecfd0..01cc7c97548 100644 --- a/docker/test/pvs/Dockerfile +++ b/docker/test/pvs/Dockerfile @@ -4,11 +4,7 @@ ARG FROM_TAG=latest FROM clickhouse/binary-builder:$FROM_TAG -# PVS studio doesn't support aarch64/arm64, so there is a check for it everywhere -# We'll produce an empty image for arm64 -ARG TARGETARCH - -RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \ +RUN apt-get update --yes \ && apt-get install \ bash \ wget \ @@ -21,7 +17,7 @@ RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \ libprotoc-dev \ libgrpc++-dev \ libc-ares-dev \ - --yes --no-install-recommends ) + --yes --no-install-recommends #RUN wget -nv -O - http://files.viva64.com/etc/pubkey.txt | sudo apt-key add - #RUN sudo wget -nv -O /etc/apt/sources.list.d/viva64.list http://files.viva64.com/etc/viva64.list @@ -33,7 +29,7 @@ RUN test x$TARGETARCH = xarm64 || ( apt-get update --yes \ ENV PKG_VERSION="pvs-studio-latest" -RUN test x$TARGETARCH = xarm64 || ( set -x \ +RUN set -x \ && export PUBKEY_HASHSUM="ad369a2e9d8b8c30f5a9f2eb131121739b79c78e03fef0f016ea51871a5f78cd4e6257b270dca0ac3be3d1f19d885516" \ && wget -nv https://files.viva64.com/etc/pubkey.txt -O /tmp/pubkey.txt \ && echo "${PUBKEY_HASHSUM} /tmp/pubkey.txt" | sha384sum -c \ @@ -41,7 +37,7 @@ RUN test x$TARGETARCH = xarm64 || ( set -x \ && wget -nv "https://files.viva64.com/${PKG_VERSION}.deb" \ && { debsig-verify ${PKG_VERSION}.deb \ || echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \ - && dpkg -i "${PKG_VERSION}.deb" ) + && dpkg -i "${PKG_VERSION}.deb" ENV CCACHE_DIR=/test_output/ccache diff --git a/docker/test/stateless/run.sh b/docker/test/stateless/run.sh index d6d9f189e89..7c2563f2d86 100755 --- a/docker/test/stateless/run.sh +++ b/docker/test/stateless/run.sh @@ -12,7 +12,11 @@ dpkg -i package_folder/clickhouse-common-static_*.deb dpkg -i package_folder/clickhouse-common-static-dbg_*.deb dpkg -i package_folder/clickhouse-server_*.deb dpkg -i package_folder/clickhouse-client_*.deb -dpkg -i package_folder/clickhouse-test_*.deb +if [[ -n "$TEST_CASES_FROM_DEB" ]] && [[ "$TEST_CASES_FROM_DEB" -eq 1 ]]; then + dpkg -i package_folder/clickhouse-test_*.deb +else + ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test +fi # install test configs /usr/share/clickhouse-test/config/install.sh @@ -85,6 +89,10 @@ function run_tests() # everything in parallel except DatabaseReplicated. See below. fi + if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then + ADDITIONAL_OPTIONS+=('--s3-storage') + fi + if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then ADDITIONAL_OPTIONS+=('--replicated-database') ADDITIONAL_OPTIONS+=('--jobs') diff --git a/docker/test/stress/run.sh b/docker/test/stress/run.sh index 4387d16ea7c..e57dbc38ded 100755 --- a/docker/test/stress/run.sh +++ b/docker/test/stress/run.sh @@ -148,14 +148,12 @@ info signals continue gcore backtrace full -info locals +thread apply all backtrace full info registers disassemble /s up -info locals disassemble /s up -info locals disassemble /s p \"done\" detach @@ -269,5 +267,5 @@ clickhouse-local --structure "test String, res String" -q "SELECT 'failure', tes # Default filename is 'core.PROCESS_ID' for core in core.*; do pigz $core - mv $core.gz /output/ + mv $core.gz /test_output/ done diff --git a/docker/test/style/Dockerfile b/docker/test/style/Dockerfile index a68b52170e0..85c751edfbe 100644 --- a/docker/test/style/Dockerfile +++ b/docker/test/style/Dockerfile @@ -11,6 +11,7 @@ RUN apt-get update && env DEBIAN_FRONTEND=noninteractive apt-get install --yes \ curl \ git \ libxml2-utils \ + moreutils \ pylint \ python3-pip \ shellcheck \ diff --git a/docker/test/style/process_style_check_result.py b/docker/test/style/process_style_check_result.py index b7e00c49e06..655b7d70243 100755 --- a/docker/test/style/process_style_check_result.py +++ b/docker/test/style/process_style_check_result.py @@ -10,72 +10,26 @@ def process_result(result_folder): status = "success" description = "" test_results = [] + checks = ( + ("header duplicates", "duplicate_output.txt"), + ("shellcheck", "shellcheck_output.txt"), + ("style", "style_output.txt"), + ("typos", "typos_output.txt"), + ("whitespaces", "whitespaces_output.txt"), + ("workflows", "workflows_output.txt"), + ) - duplicate_log_path = "{}/duplicate_output.txt".format(result_folder) - if not os.path.exists(duplicate_log_path): - logging.info("No header duplicates check log on path %s", duplicate_log_path) - return "exception", "No header duplicates check log", [] - elif os.stat(duplicate_log_path).st_size != 0: - description += " Header duplicates check failed. " - test_results.append(("Header duplicates check", "FAIL")) - status = "failure" - else: - test_results.append(("Header duplicates check", "OK")) - - shellcheck_log_path = "{}/shellcheck_output.txt".format(result_folder) - if not os.path.exists(shellcheck_log_path): - logging.info("No shellcheck log on path %s", shellcheck_log_path) - return "exception", "No shellcheck log", [] - elif os.stat(shellcheck_log_path).st_size != 0: - description += " Shellcheck check failed. " - test_results.append(("Shellcheck ", "FAIL")) - status = "failure" - else: - test_results.append(("Shellcheck", "OK")) - - style_log_path = "{}/style_output.txt".format(result_folder) - if not os.path.exists(style_log_path): - logging.info("No style check log on path %s", style_log_path) - return "exception", "No style check log", [] - elif os.stat(style_log_path).st_size != 0: - description += "Style check failed. " - test_results.append(("Style check", "FAIL")) - status = "failure" - else: - test_results.append(("Style check", "OK")) - - typos_log_path = "{}/typos_output.txt".format(result_folder) - if not os.path.exists(typos_log_path): - logging.info("No typos check log on path %s", typos_log_path) - return "exception", "No typos check log", [] - elif os.stat(typos_log_path).st_size != 0: - description += "Typos check failed. " - test_results.append(("Typos check", "FAIL")) - status = "failure" - else: - test_results.append(("Typos check", "OK")) - - whitespaces_log_path = "{}/whitespaces_output.txt".format(result_folder) - if not os.path.exists(whitespaces_log_path): - logging.info("No whitespaces check log on path %s", whitespaces_log_path) - return "exception", "No whitespaces check log", [] - elif os.stat(whitespaces_log_path).st_size != 0: - description += "Whitespaces check failed. " - test_results.append(("Whitespaces check", "FAIL")) - status = "failure" - else: - test_results.append(("Whitespaces check", "OK")) - - workflows_log_path = "{}/workflows_output.txt".format(result_folder) - if not os.path.exists(workflows_log_path): - logging.info("No workflows check log on path %s", style_log_path) - return "exception", "No workflows check log", [] - elif os.stat(whitespaces_log_path).st_size != 0: - description += "Workflows check failed. " - test_results.append(("Workflows check", "FAIL")) - status = "failure" - else: - test_results.append(("Workflows check", "OK")) + for name, out_file in checks: + full_path = os.path.join(result_folder, out_file) + if not os.path.exists(full_path): + logging.info("No %s check log on path %s", name, full_path) + return "exception", f"No {name} check log", [] + elif os.stat(full_path).st_size != 0: + description += f"Check {name} failed. " + test_results.append((f"Check {name}", "FAIL")) + status = "failure" + else: + test_results.append((f"Check {name}", "OK")) if not description: description += "Style check success" diff --git a/docker/test/style/run.sh b/docker/test/style/run.sh index 98bc0053ab9..ce3ea4e50a6 100755 --- a/docker/test/style/run.sh +++ b/docker/test/style/run.sh @@ -3,10 +3,16 @@ # yaml check is not the best one cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv +echo "Check duplicates" | ts ./check-duplicate-includes.sh |& tee /test_output/duplicate_output.txt +echo "Check style" | ts ./check-style -n |& tee /test_output/style_output.txt +echo "Check typos" | ts ./check-typos |& tee /test_output/typos_output.txt +echo "Check whitespaces" | ts ./check-whitespaces -n |& tee /test_output/whitespaces_output.txt +echo "Check sorkflows" | ts ./check-workflows |& tee /test_output/workflows_output.txt +echo "Check shell scripts with shellcheck" | ts ./shellcheck-run.sh |& tee /test_output/shellcheck_output.txt /process_style_check_result.py || echo -e "failure\tCannot parse results" > /test_output/check_status.tsv diff --git a/docker/test/testflows/runner/Dockerfile b/docker/test/testflows/runner/Dockerfile index d15f237587b..fbff6fd5e97 100644 --- a/docker/test/testflows/runner/Dockerfile +++ b/docker/test/testflows/runner/Dockerfile @@ -43,24 +43,27 @@ RUN pip3 install urllib3 testflows==1.7.20 docker-compose==1.29.1 docker==5.0.0 ENV DOCKER_CHANNEL stable ENV DOCKER_VERSION 20.10.6 -RUN set -eux; \ - \ -# this "case" statement is generated via "update.sh" - \ - if ! wget -nv -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/x86_64/docker-${DOCKER_VERSION}.tgz"; then \ - echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${x86_64}'"; \ - exit 1; \ - fi; \ - \ - tar --extract \ +# Architecture of the image when BuildKit/buildx is used +ARG TARGETARCH + +# Install docker +RUN arch=${TARGETARCH:-amd64} \ + && case $arch in \ + amd64) rarch=x86_64 ;; \ + arm64) rarch=aarch64 ;; \ + esac \ + && set -eux \ + && if ! wget -nv -O docker.tgz "https://download.docker.com/linux/static/${DOCKER_CHANNEL}/${rarch}/docker-${DOCKER_VERSION}.tgz"; then \ + echo >&2 "error: failed to download 'docker-${DOCKER_VERSION}' from '${DOCKER_CHANNEL}' for '${rarch}'" \ + && exit 1; \ + fi \ + && tar --extract \ --file docker.tgz \ --strip-components 1 \ --directory /usr/local/bin/ \ - ; \ - rm docker.tgz; \ - \ - dockerd --version; \ - docker --version + && rm docker.tgz \ + && dockerd --version \ + && docker --version COPY modprobe.sh /usr/local/bin/modprobe COPY dockerd-entrypoint.sh /usr/local/bin/ diff --git a/docs/_includes/cmake_in_clickhouse_header.md b/docs/_includes/cmake_in_clickhouse_header.md index f950cdcc6db..02019f13964 100644 --- a/docs/_includes/cmake_in_clickhouse_header.md +++ b/docs/_includes/cmake_in_clickhouse_header.md @@ -22,7 +22,7 @@ cmake .. \ 1. ClickHouse's source CMake files (located in the root directory and in `/src`). 2. Arch-dependent CMake files (located in `/cmake/*os_name*`). -3. Libraries finders (search for contrib libraries, located in `/cmake/find`). +3. Libraries finders (search for contrib libraries, located in `/contrib/*/CMakeLists.txt`). 3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`) ## List of CMake flags diff --git a/docs/_includes/install/deb.sh b/docs/_includes/install/deb.sh index 7dcca601d33..21106e9fc47 100644 --- a/docs/_includes/install/deb.sh +++ b/docs/_includes/install/deb.sh @@ -8,4 +8,4 @@ sudo apt-get update sudo apt-get install -y clickhouse-server clickhouse-client sudo service clickhouse-server start -clickhouse-client +clickhouse-client # or "clickhouse-client --password" if you set up a password. diff --git a/docs/_includes/install/rpm.sh b/docs/_includes/install/rpm.sh index de4a07420f7..e3fd1232047 100644 --- a/docs/_includes/install/rpm.sh +++ b/docs/_includes/install/rpm.sh @@ -4,4 +4,4 @@ sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/clickhouse.re sudo yum install clickhouse-server clickhouse-client sudo /etc/init.d/clickhouse-server start -clickhouse-client +clickhouse-client # or "clickhouse-client --password" if you set up a password. diff --git a/docs/en/development/developer-instruction.md b/docs/en/development/developer-instruction.md index ccf6da355b9..f7d7100d181 100644 --- a/docs/en/development/developer-instruction.md +++ b/docs/en/development/developer-instruction.md @@ -125,10 +125,6 @@ For installing CMake and Ninja on Mac OS X first install Homebrew and then insta Next, check the version of CMake: `cmake --version`. If it is below 3.12, you should install a newer version from the website: https://cmake.org/download/. -## Optional External Libraries {#optional-external-libraries} - -ClickHouse uses several external libraries for building. All of them do not need to be installed separately as they are built together with ClickHouse from the sources located in the submodules. You can check the list in `contrib`. - ## C++ Compiler {#c-compiler} Compilers Clang starting from version 11 is supported for building ClickHouse. diff --git a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md index 79ba06096b2..e1d571c909c 100644 --- a/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/graphitemergetree.md @@ -97,13 +97,16 @@ Structure of the `patterns` section: ``` text pattern + rule_type regexp function pattern + rule_type regexp age + precision ... pattern + rule_type regexp function age + precision @@ -127,12 +130,20 @@ When processing a row, ClickHouse checks the rules in the `pattern` sections. Ea Fields for `pattern` and `default` sections: -- `regexp`– A pattern for the metric name. +- `rule_type` - a rule's type. It's applied only to a particular metrics. The engine use it to separate plain and tagged metrics. Optional parameter. Default value: `all`. +It's unnecessary when performance is not critical, or only one metrics type is used, e.g. plain metrics. By default only one type of rules set is created. Otherwise, if any of special types is defined, two different sets are created. One for plain metrics (root.branch.leaf) and one for tagged metrics (root.branch.leaf;tag1=value1). +The default rules are ended up in both sets. +Valid values: + - `all` (default) - a universal rule, used when `rule_type` is omitted. + - `plain` - a rule for plain metrics. The field `regexp` is processed as regular expression. + - `tagged` - a rule for tagged metrics (metrics are stored in DB in the format of `someName?tag1=value1&tag2=value2&tag3=value3`). Regular expression must be sorted by tags' names, first tag must be `__name__` if exists. The field `regexp` is processed as regular expression. + - `tag_list` - a rule for tagged matrics, a simple DSL for easier metric description in graphite format `someName;tag1=value1;tag2=value2`, `someName`, or `tag1=value1;tag2=value2`. The field `regexp` is translated into a `tagged` rule. The sorting by tags' names is unnecessary, ti will be done automatically. A tag's value (but not a name) can be set as a regular expression, e.g. `env=(dev|staging)`. +- `regexp` – A pattern for the metric name (a regular or DSL). - `age` – The minimum age of the data in seconds. - `precision`– How precisely to define the age of the data in seconds. Should be a divisor for 86400 (seconds in a day). - `function` – The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. Accepted functions: min / max / any / avg. The average is calculated imprecisely, like the average of the averages. -### Configuration Example {#configuration-example} +### Configuration Example without rules types {#configuration-example} ``` xml @@ -167,6 +178,81 @@ Fields for `pattern` and `default` sections: ``` +### Configuration Example with rules types {#configuration-typed-example} + +``` xml + + Version + + plain + click_cost + any + + 0 + 5 + + + 86400 + 60 + + + + tagged + ^((.*)|.)min\? + min + + 0 + 5 + + + 86400 + 60 + + + + tagged + + min + + 0 + 5 + + + 86400 + 60 + + + + tag_list + someName;tag2=value2 + + 0 + 5 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + +``` + + !!! warning "Warning" Data rollup is performed during merges. Usually, for old partitions, merges are not started, so for rollup it is necessary to trigger an unscheduled merge using [optimize](../../../sql-reference/statements/optimize.md). Or use additional tools, for example [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). diff --git a/docs/en/engines/table-engines/mergetree-family/mergetree.md b/docs/en/engines/table-engines/mergetree-family/mergetree.md index 6769f48a466..92865c94475 100644 --- a/docs/en/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/en/engines/table-engines/mergetree-family/mergetree.md @@ -886,3 +886,12 @@ S3 disk can be configured as `main` or `cold` storage: ``` In case of `cold` option a data can be moved to S3 if local disk free size will be smaller than `move_factor * disk_size` or by TTL move rule. + +## Virtual Columns {#virtual-columns} + +- `_part` — Name of a part. +- `_part_index` — Sequential index of the part in the query result. +- `_partition_id` — Name of a partition. +- `_part_uuid` — Unique part identifier (if enabled MergeTree setting `assign_part_uuids`). +- `_partition_value` — Values (a tuple) of a `partition by` expression. +- `_sample_factor` — Sample factor (from the query). diff --git a/docs/en/engines/table-engines/special/buffer.md b/docs/en/engines/table-engines/special/buffer.md index 884774cbfae..d1f92d347a4 100644 --- a/docs/en/engines/table-engines/special/buffer.md +++ b/docs/en/engines/table-engines/special/buffer.md @@ -54,10 +54,8 @@ If the set of columns in the Buffer table does not match the set of columns in a If the types do not match for one of the columns in the Buffer table and a subordinate table, an error message is entered in the server log, and the buffer is cleared. The same thing happens if the subordinate table does not exist when the buffer is flushed. -If you need to run ALTER for a subordinate table, and the Buffer table, we recommend first deleting the Buffer table, running ALTER for the subordinate table, then creating the Buffer table again. - !!! attention "Attention" - Running ALTER on the Buffer table in releases made before 28 Sep 2020 will cause a `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), so deleting the Buffer table and then recreating is the only option. It is advisable to check that this error is fixed in your release before trying to run ALTER on the Buffer table. + Running ALTER on the Buffer table in releases made before 26 Oct 2021 will cause a `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117) and [#30565](https://github.com/ClickHouse/ClickHouse/pull/30565)), so deleting the Buffer table and then recreating is the only option. It is advisable to check that this error is fixed in your release before trying to run ALTER on the Buffer table. If the server is restarted abnormally, the data in the buffer is lost. diff --git a/docs/en/engines/table-engines/special/distributed.md b/docs/en/engines/table-engines/special/distributed.md index faa1026b919..4d2454298f2 100644 --- a/docs/en/engines/table-engines/special/distributed.md +++ b/docs/en/engines/table-engines/special/distributed.md @@ -209,6 +209,8 @@ When querying a `Distributed` table, `SELECT` queries are sent to all shards and When the `max_parallel_replicas` option is enabled, query processing is parallelized across all replicas within a single shard. For more information, see the section [max_parallel_replicas](../../../operations/settings/settings.md#settings-max_parallel_replicas). +To learn more about how distibuted `in` and `global in` queries are processed, refer to [this](../../../sql-reference/operators/in.md#select-distributed-subqueries) documentation. + ## Virtual Columns {#virtual-columns} - `_shard_num` — Contains the `shard_num` value from the table `system.clusters`. Type: [UInt32](../../../sql-reference/data-types/int-uint.md). diff --git a/docs/en/engines/table-engines/special/url.md b/docs/en/engines/table-engines/special/url.md index 04f035206b5..26d928085ce 100644 --- a/docs/en/engines/table-engines/special/url.md +++ b/docs/en/engines/table-engines/special/url.md @@ -7,18 +7,29 @@ toc_title: URL Queries data to/from a remote HTTP/HTTPS server. This engine is similar to the [File](../../../engines/table-engines/special/file.md) engine. -Syntax: `URL(URL, Format)` +Syntax: `URL(URL [,Format] [,CompressionMethod])` + +- The `URL` parameter must conform to the structure of a Uniform Resource Locator. The specified URL must point to a server that uses HTTP or HTTPS. This does not require any additional headers for getting a response from the server. + +- The `Format` must be one that ClickHouse can use in `SELECT` queries and, if necessary, in `INSERTs`. For the full list of supported formats, see [Formats](../../../interfaces/formats.md#formats). + +- `CompressionMethod` indicates that whether the HTTP body should be compressed. If the compression is enabled, the HTTP packets sent by the URL engine contain 'Content-Encoding' header to indicate which compression method is used. + +To enable compression, please first make sure the remote HTTP endpoint indicated by the `URL` parameter supports corresponding compression algorithm. + +The supported `CompressionMethod` should be one of following: +- gzip or gz +- deflate +- brotli or br +- lzma or xz +- zstd or zst +- lz4 +- bz2 +- snappy +- none ## Usage {#using-the-engine-in-the-clickhouse-server} -The `format` must be one that ClickHouse can use in -`SELECT` queries and, if necessary, in `INSERTs`. For the full list of supported formats, see -[Formats](../../../interfaces/formats.md#formats). - -The `URL` must conform to the structure of a Uniform Resource Locator. The specified URL must point to a server -that uses HTTP or HTTPS. This does not require any -additional headers for getting a response from the server. - `INSERT` and `SELECT` queries are transformed to `POST` and `GET` requests, respectively. For processing `POST` requests, the remote server must support [Chunked transfer encoding](https://en.wikipedia.org/wiki/Chunked_transfer_encoding). diff --git a/docs/en/faq/index.md b/docs/en/faq/index.md index d845b8c5898..891e1ea464e 100644 --- a/docs/en/faq/index.md +++ b/docs/en/faq/index.md @@ -25,6 +25,7 @@ Categories: - **[Operations](../faq/operations/index.md)** - [Which ClickHouse version to use in production?](../faq/operations/production.md) - [Is it possible to delete old records from a ClickHouse table?](../faq/operations/delete-old-data.md) + - [Does ClickHouse support multi-region replication?](../faq/operations/multi-region-replication.md) - **[Integration](../faq/integration/index.md)** - [How do I export data from ClickHouse to a file?](../faq/integration/file-export.md) - [What if I have a problem with encodings when connecting to Oracle via ODBC?](../faq/integration/oracle-odbc.md) diff --git a/docs/en/getting-started/install.md b/docs/en/getting-started/install.md index 70a1b8349ff..c03daf45b02 100644 --- a/docs/en/getting-started/install.md +++ b/docs/en/getting-started/install.md @@ -69,14 +69,14 @@ You can also download and install packages manually from [here](https://repo.cli It is recommended to use official pre-compiled `tgz` archives for all Linux distributions, where installation of `deb` or `rpm` packages is not possible. The required version can be downloaded with `curl` or `wget` from repository https://repo.clickhouse.com/tgz/. -After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest version: +After that downloaded archives should be unpacked and installed with installation scripts. Example for the latest stable version: ``` bash -export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-common-static-dbg-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-server-$LATEST_VERSION.tgz -curl -O https://repo.clickhouse.com/tgz/clickhouse-client-$LATEST_VERSION.tgz +export LATEST_VERSION=`curl https://api.github.com/repos/ClickHouse/ClickHouse/tags 2>/dev/null | grep stable | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+' | head -n 1` +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-common-static-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-common-static-dbg-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-server-$LATEST_VERSION.tgz +curl -O https://repo.clickhouse.com/tgz/stable/clickhouse-client-$LATEST_VERSION.tgz tar -xzvf clickhouse-common-static-$LATEST_VERSION.tgz sudo clickhouse-common-static-$LATEST_VERSION/install/doinst.sh diff --git a/docs/en/interfaces/http.md b/docs/en/interfaces/http.md index f8f6f26d208..d72fb4d6f17 100644 --- a/docs/en/interfaces/http.md +++ b/docs/en/interfaces/http.md @@ -23,11 +23,13 @@ Web UI can be accessed here: `http://localhost:8123/play`. ![Web UI](../images/play.png) -In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. +In health-check scripts use `GET /ping` request. This handler always returns “Ok.” (with a line feed at the end). Available from version 18.12.13. See also `/replicas_status` to check replica's delay. ``` bash $ curl 'http://localhost:8123/ping' Ok. +$ curl 'http://localhost:8123/replicas_status' +Ok. ``` Send the request as a URL ‘query’ parameter, or as a POST. Or send the beginning of the query in the ‘query’ parameter, and the rest in the POST (we’ll explain later why this is necessary). The size of the URL is limited to 16 KB, so keep this in mind when sending large queries. diff --git a/docs/en/introduction/adopters.md b/docs/en/introduction/adopters.md index 5efa1b971bc..9c7fab7424d 100644 --- a/docs/en/introduction/adopters.md +++ b/docs/en/introduction/adopters.md @@ -67,6 +67,7 @@ toc_title: Adopters | Geniee | Ad network | Main product | — | — | [Blog post in Japanese, July 2017](https://tech.geniee.co.jp/entry/2017/07/20/160100) | | Genotek | Bioinformatics | Main product | — | — | [Video, August 2020](https://youtu.be/v3KyZbz9lEE) | | Gigapipe | Managed ClickHouse | Main product | — | — | [Official website](https://gigapipe.com/) | +| Gigasheet | Analytics | Main product | — | — | Direct Reference, February 2022| | Glaber | Monitoring | Main product | — | — | [Website](https://glaber.io/) | | GraphCDN | CDN | Traffic Analytics | — | — | [Blog Post in English, August 2021](https://altinity.com/blog/delivering-insight-on-graphql-apis-with-clickhouse-at-graphcdn/) | | Grouparoo | Data Warehouse Integrations | Main product | — | — | [Official Website, November 2021](https://www.grouparoo.com/integrations) | diff --git a/docs/en/operations/clickhouse-keeper.md b/docs/en/operations/clickhouse-keeper.md index fcfc675f9d7..35ec5d858f5 100644 --- a/docs/en/operations/clickhouse-keeper.md +++ b/docs/en/operations/clickhouse-keeper.md @@ -108,7 +108,13 @@ Examples of configuration for quorum with three nodes can be found in [integrati ClickHouse Keeper is bundled into the ClickHouse server package, just add configuration of `` and start ClickHouse server as always. If you want to run standalone ClickHouse Keeper you can start it in a similar way with: ```bash -clickhouse-keeper --config /etc/your_path_to_config/config.xml --daemon +clickhouse-keeper --config /etc/your_path_to_config/config.xml +``` + +If you don't have the symlink (`clickhouse-keeper`) you can create it or specify `keeper` as argument: + +```bash +clickhouse keeper --config /etc/your_path_to_config/config.xml ``` ## Four Letter Word Commands {#four-letter-word-commands} diff --git a/docs/en/operations/opentelemetry.md b/docs/en/operations/opentelemetry.md index 6dac8736372..ec27ecfd6b2 100644 --- a/docs/en/operations/opentelemetry.md +++ b/docs/en/operations/opentelemetry.md @@ -14,7 +14,7 @@ toc_title: OpenTelemetry Support ClickHouse accepts trace context HTTP headers, as described by the [W3C recommendation](https://www.w3.org/TR/trace-context/). It also accepts trace context over a native protocol that is used for communication between ClickHouse servers or between the client and server. For manual testing, trace context headers conforming to the Trace Context recommendation can be supplied to `clickhouse-client` using `--opentelemetry-traceparent` and `--opentelemetry-tracestate` flags. -If no parent trace context is supplied, ClickHouse can start a new trace, with probability controlled by the [opentelemetry_start_trace_probability](../operations/settings/settings.md#opentelemetry-start-trace-probability) setting. +If no parent trace context is supplied or the provided trace context does not comply with W3C standard above, ClickHouse can start a new trace, with probability controlled by the [opentelemetry_start_trace_probability](../operations/settings/settings.md#opentelemetry-start-trace-probability) setting. ## Propagating the Trace Context @@ -46,8 +46,8 @@ ENGINE = URL('http://127.0.0.1:9411/api/v2/spans', 'JSONEachRow') SETTINGS output_format_json_named_tuples_as_objects = 1, output_format_json_array_of_rows = 1 AS SELECT - lower(hex(reinterpretAsFixedString(trace_id))) AS traceId, - lower(hex(parent_span_id)) AS parentId, + lower(hex(trace_id)) AS traceId, + case when parent_span_id = 0 then '' else lower(hex(parent_span_id)) end AS parentId, lower(hex(span_id)) AS id, operation_name AS name, start_time_us AS timestamp, diff --git a/docs/en/operations/optimizing-performance/sampling-query-profiler.md b/docs/en/operations/optimizing-performance/sampling-query-profiler.md index 9244592d515..72cfa59b8b2 100644 --- a/docs/en/operations/optimizing-performance/sampling-query-profiler.md +++ b/docs/en/operations/optimizing-performance/sampling-query-profiler.md @@ -27,7 +27,7 @@ To analyze the `trace_log` system table: For security reasons, introspection functions are disabled by default. -- Use the `addressToLine`, `addressToSymbol` and `demangle` [introspection functions](../../sql-reference/functions/introspection.md) to get function names and their positions in ClickHouse code. To get a profile for some query, you need to aggregate data from the `trace_log` table. You can aggregate data by individual functions or by the whole stack traces. +- Use the `addressToLine`, `addressToLineWithInlines`, `addressToSymbol` and `demangle` [introspection functions](../../sql-reference/functions/introspection.md) to get function names and their positions in ClickHouse code. To get a profile for some query, you need to aggregate data from the `trace_log` table. You can aggregate data by individual functions or by the whole stack traces. If you need to visualize `trace_log` info, try [flamegraph](../../interfaces/third-party/gui/#clickhouse-flamegraph) and [speedscope](https://github.com/laplab/clickhouse-speedscope). diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 8a0fd618d32..56f1e7fe3bb 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -1803,6 +1803,48 @@ If an INSERTed block is skipped due to deduplication in the source table, there At the same time, this behaviour “breaks” `INSERT` idempotency. If an `INSERT` into the main table was successful and `INSERT` into a materialized view failed (e.g. because of communication failure with Zookeeper) a client will get an error and can retry the operation. However, the materialized view won’t receive the second insert because it will be discarded by deduplication in the main (source) table. The setting `deduplicate_blocks_in_dependent_materialized_views` allows for changing this behaviour. On retry, a materialized view will receive the repeat insert and will perform a deduplication check by itself, ignoring check result for the source table, and will insert rows lost because of the first failure. +## insert_deduplication_token {#insert_deduplication_token} + +The setting allows a user to provide own deduplication semantic in MergeTree/ReplicatedMergeTree +For example, by providing a unique value for the setting in each INSERT statement, +user can avoid the same inserted data being deduplicated. + +Possilbe values: + +- Any string + +Default value: empty string (disabled) + +`insert_deduplication_token` is used for deduplication _only_ when not empty. + +Example: + +```sql +CREATE TABLE test_table +( A Int64 ) +ENGINE = MergeTree +ORDER BY A +SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test' (1); + +-- the next insert won't be deduplicated because insert_deduplication_token is different +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test1' (1); + +-- the next insert will be deduplicated because insert_deduplication_token +-- is the same as one of the previous +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test' (2); + +SELECT * FROM test_table + +┌─A─┐ +│ 1 │ +└───┘ +┌─A─┐ +│ 1 │ +└───┘ +``` + ## max_network_bytes {#settings-max-network-bytes} Limits the data volume (in bytes) that is received or transmitted over the network when executing a query. This setting applies to every individual query. @@ -2304,7 +2346,7 @@ Possible values: - 1 — Enabled. - 0 — Disabled. -Default value: `0`. +Default value: `1`. ## output_format_parallel_formatting {#output-format-parallel-formatting} @@ -2315,7 +2357,7 @@ Possible values: - 1 — Enabled. - 0 — Disabled. -Default value: `0`. +Default value: `1`. ## min_chunk_bytes_for_parallel_parsing {#min-chunk-bytes-for-parallel-parsing} diff --git a/docs/en/operations/system-tables/stack_trace.md b/docs/en/operations/system-tables/stack_trace.md index eb1824a6f66..e2135e4beb6 100644 --- a/docs/en/operations/system-tables/stack_trace.md +++ b/docs/en/operations/system-tables/stack_trace.md @@ -2,7 +2,7 @@ Contains stack traces of all server threads. Allows developers to introspect the server state. -To analyze stack frames, use the `addressToLine`, `addressToSymbol` and `demangle` [introspection functions](../../sql-reference/functions/introspection.md). +To analyze stack frames, use the `addressToLine`, `addressToLineWithInlines`, `addressToSymbol` and `demangle` [introspection functions](../../sql-reference/functions/introspection.md). Columns: diff --git a/docs/en/operations/system-tables/trace_log.md b/docs/en/operations/system-tables/trace_log.md index 4902b09004d..ab08ef7415c 100644 --- a/docs/en/operations/system-tables/trace_log.md +++ b/docs/en/operations/system-tables/trace_log.md @@ -4,7 +4,7 @@ Contains stack traces collected by the sampling query profiler. ClickHouse creates this table when the [trace_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-trace_log) server configuration section is set. Also the [query_profiler_real_time_period_ns](../../operations/settings/settings.md#query_profiler_real_time_period_ns) and [query_profiler_cpu_time_period_ns](../../operations/settings/settings.md#query_profiler_cpu_time_period_ns) settings should be set. -To analyze logs, use the `addressToLine`, `addressToSymbol` and `demangle` introspection functions. +To analyze logs, use the `addressToLine`, `addressToLineWithInlines`, `addressToSymbol` and `demangle` introspection functions. Columns: diff --git a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md index fd391298bc3..7d8d255e15b 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/studentttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/studentttest.md @@ -10,7 +10,7 @@ Applies Student's t-test to samples from two populations. **Syntax** ``` sql -studentTTest(sample_data, sample_index) +studentTTest([confidence_level])(sample_data, sample_index) ``` Values of both samples are in the `sample_data` column. If `sample_index` equals to 0 then the value in that row belongs to the sample from the first population. Otherwise it belongs to the sample from the second population. @@ -21,12 +21,19 @@ The null hypothesis is that means of populations are equal. Normal distribution - `sample_data` — Sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). - `sample_index` — Sample index. [Integer](../../../sql-reference/data-types/int-uint.md). +**Parameters** + +- `confidence_level` — Confidence level in order to calculate confidence intervals. [Float](../../../sql-reference/data-types/float.md). + + **Returned values** -[Tuple](../../../sql-reference/data-types/tuple.md) with two elements: +[Tuple](../../../sql-reference/data-types/tuple.md) with two or four elements (if the optional `confidence_level` is specified): - calculated t-statistic. [Float64](../../../sql-reference/data-types/float.md). - calculated p-value. [Float64](../../../sql-reference/data-types/float.md). +- [calculated confidence-interval-low.] [Float64](../../../sql-reference/data-types/float.md). +- [calculated confidence-interval-high.] [Float64](../../../sql-reference/data-types/float.md). **Example** diff --git a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md index 62f5761b32e..2e127f87f9f 100644 --- a/docs/en/sql-reference/aggregate-functions/reference/welchttest.md +++ b/docs/en/sql-reference/aggregate-functions/reference/welchttest.md @@ -10,7 +10,7 @@ Applies Welch's t-test to samples from two populations. **Syntax** ``` sql -welchTTest(sample_data, sample_index) +welchTTest([confidence_level])(sample_data, sample_index) ``` Values of both samples are in the `sample_data` column. If `sample_index` equals to 0 then the value in that row belongs to the sample from the first population. Otherwise it belongs to the sample from the second population. @@ -21,12 +21,18 @@ The null hypothesis is that means of populations are equal. Normal distribution - `sample_data` — Sample data. [Integer](../../../sql-reference/data-types/int-uint.md), [Float](../../../sql-reference/data-types/float.md) or [Decimal](../../../sql-reference/data-types/decimal.md). - `sample_index` — Sample index. [Integer](../../../sql-reference/data-types/int-uint.md). +**Parameters** + +- `confidence_level` — Confidence level in order to calculate confidence intervals. [Float](../../../sql-reference/data-types/float.md). + **Returned values** -[Tuple](../../../sql-reference/data-types/tuple.md) with two elements: +[Tuple](../../../sql-reference/data-types/tuple.md) with two or four elements (if the optional `confidence_level` is specified) - calculated t-statistic. [Float64](../../../sql-reference/data-types/float.md). - calculated p-value. [Float64](../../../sql-reference/data-types/float.md). +- [calculated confidence-interval-low.] [Float64](../../../sql-reference/data-types/float.md). +- [calculated confidence-interval-high.] [Float64](../../../sql-reference/data-types/float.md). **Example** diff --git a/docs/en/sql-reference/data-types/int-uint.md b/docs/en/sql-reference/data-types/int-uint.md index 588b5a2d7d6..4cc590d9fa5 100644 --- a/docs/en/sql-reference/data-types/int-uint.md +++ b/docs/en/sql-reference/data-types/int-uint.md @@ -1,9 +1,9 @@ --- toc_priority: 40 -toc_title: UInt8, UInt16, UInt32, UInt64, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 +toc_title: UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 --- -# UInt8, UInt16, UInt32, UInt64, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 {#uint8-uint16-uint32-uint64-uint256-int8-int16-int32-int64-int128-int256} +# UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 Fixed-length integers, with or without a sign. diff --git a/docs/en/sql-reference/functions/functions-for-nulls.md b/docs/en/sql-reference/functions/functions-for-nulls.md index 29de9ee4b70..42307093dda 100644 --- a/docs/en/sql-reference/functions/functions-for-nulls.md +++ b/docs/en/sql-reference/functions/functions-for-nulls.md @@ -120,7 +120,7 @@ The `mail` and `phone` fields are of type String, but the `icq` field is `UInt32 Get the first available contact method for the customer from the contact list: ``` sql -SELECT coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook; +SELECT name, coalesce(mail, phone, CAST(icq,'Nullable(String)')) FROM aBook; ``` ``` text diff --git a/docs/en/sql-reference/functions/introspection.md b/docs/en/sql-reference/functions/introspection.md index 21b570c65d4..1be68c6bdd4 100644 --- a/docs/en/sql-reference/functions/introspection.md +++ b/docs/en/sql-reference/functions/introspection.md @@ -113,6 +113,111 @@ trace_source_code_lines: /lib/x86_64-linux-gnu/libpthread-2.27.so /build/glibc-OTsEL5/glibc-2.27/misc/../sysdeps/unix/sysv/linux/x86_64/clone.S:97 ``` +## addressToLineWithInlines {#addresstolinewithinlines} + +Similar to `addressToLine`, but it will return an Array with all inline functions, and will be much slower as a price. + +If you use official ClickHouse packages, you need to install the `clickhouse-common-static-dbg` package. + +**Syntax** + +``` sql +addressToLineWithInlines(address_of_binary_instruction) +``` + +**Arguments** + +- `address_of_binary_instruction` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Address of instruction in a running process. + +**Returned value** + +- Array which first element is source code filename and the line number in this file delimited by colon. And from second element, inline functions' source code filename and line number and function name are listed. + +- Array with single element which is name of a binary, if the function couldn’t find the debug information. + +- Empty array, if the address is not valid. + +Type: [Array(String)](../../sql-reference/data-types/array.md). + +**Example** + +Enabling introspection functions: + +``` sql +SET allow_introspection_functions=1; +``` + +Applying the function to address. + +```sql +SELECT addressToLineWithInlines(531055181::UInt64); +``` + +``` text +┌─addressToLineWithInlines(CAST('531055181', 'UInt64'))────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ ['./src/Functions/addressToLineWithInlines.cpp:98','./build_normal_debug/./src/Functions/addressToLineWithInlines.cpp:176:DB::(anonymous namespace)::FunctionAddressToLineWithInlines::implCached(unsigned long) const'] │ +└──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ +``` + +Applying the function to the whole stack trace: + +``` sql +SELECT + ta, addressToLineWithInlines(arrayJoin(trace) as ta) +FROM system.trace_log +WHERE + query_id = '5e173544-2020-45de-b645-5deebe2aae54'; +``` + +The [arrayJoin](../../sql-reference/functions/array-functions.md#array-functions-join) functions will split array to rows. + +``` text +┌────────ta─┬─addressToLineWithInlines(arrayJoin(trace))───────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┐ +│ 365497529 │ ['./build_normal_debug/./contrib/libcxx/include/string_view:252'] │ +│ 365593602 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:191'] │ +│ 365593866 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365592528 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365591003 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:477'] │ +│ 365590479 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:442'] │ +│ 365590600 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:457'] │ +│ 365598941 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365607098 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365590571 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:451'] │ +│ 365598941 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365607098 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365590571 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:451'] │ +│ 365598941 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365607098 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365590571 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:451'] │ +│ 365598941 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:0'] │ +│ 365597289 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:807'] │ +│ 365599840 │ ['./build_normal_debug/./src/Common/Dwarf.cpp:1118'] │ +│ 531058145 │ ['./build_normal_debug/./src/Functions/addressToLineWithInlines.cpp:152'] │ +│ 531055181 │ ['./src/Functions/addressToLineWithInlines.cpp:98','./build_normal_debug/./src/Functions/addressToLineWithInlines.cpp:176:DB::(anonymous namespace)::FunctionAddressToLineWithInlines::implCached(unsigned long) const'] │ +│ 422333613 │ ['./build_normal_debug/./src/Functions/IFunctionAdaptors.h:21'] │ +│ 586866022 │ ['./build_normal_debug/./src/Functions/IFunction.cpp:216'] │ +│ 586869053 │ ['./build_normal_debug/./src/Functions/IFunction.cpp:264'] │ +│ 586873237 │ ['./build_normal_debug/./src/Functions/IFunction.cpp:334'] │ +│ 597901620 │ ['./build_normal_debug/./src/Interpreters/ExpressionActions.cpp:601'] │ +│ 597898534 │ ['./build_normal_debug/./src/Interpreters/ExpressionActions.cpp:718'] │ +│ 630442912 │ ['./build_normal_debug/./src/Processors/Transforms/ExpressionTransform.cpp:23'] │ +│ 546354050 │ ['./build_normal_debug/./src/Processors/ISimpleTransform.h:38'] │ +│ 626026993 │ ['./build_normal_debug/./src/Processors/ISimpleTransform.cpp:89'] │ +│ 626294022 │ ['./build_normal_debug/./src/Processors/Executors/ExecutionThreadContext.cpp:45'] │ +│ 626293730 │ ['./build_normal_debug/./src/Processors/Executors/ExecutionThreadContext.cpp:63'] │ +│ 626169525 │ ['./build_normal_debug/./src/Processors/Executors/PipelineExecutor.cpp:213'] │ +│ 626170308 │ ['./build_normal_debug/./src/Processors/Executors/PipelineExecutor.cpp:178'] │ +│ 626166348 │ ['./build_normal_debug/./src/Processors/Executors/PipelineExecutor.cpp:329'] │ +│ 626163461 │ ['./build_normal_debug/./src/Processors/Executors/PipelineExecutor.cpp:84'] │ +│ 626323536 │ ['./build_normal_debug/./src/Processors/Executors/PullingAsyncPipelineExecutor.cpp:85'] │ +│ 626323277 │ ['./build_normal_debug/./src/Processors/Executors/PullingAsyncPipelineExecutor.cpp:112'] │ +│ 626323133 │ ['./build_normal_debug/./contrib/libcxx/include/type_traits:3682'] │ +│ 626323041 │ ['./build_normal_debug/./contrib/libcxx/include/tuple:1415'] │ +└───────────┴──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────┘ + +``` + + ## addressToSymbol {#addresstosymbol} Converts virtual memory address inside ClickHouse server process to the symbol from ClickHouse object files. diff --git a/docs/en/sql-reference/functions/tuple-functions.md b/docs/en/sql-reference/functions/tuple-functions.md index 8502fcdcf66..96bceb8958c 100644 --- a/docs/en/sql-reference/functions/tuple-functions.md +++ b/docs/en/sql-reference/functions/tuple-functions.md @@ -22,7 +22,7 @@ tuple(x, y, …) ## tupleElement {#tupleelement} A function that allows getting a column from a tuple. -‘N’ is the column index, starting from 1. N must be a constant. ‘N’ must be a constant. ‘N’ must be a strict postive integer no greater than the size of the tuple. +‘N’ is the column index, starting from 1. ‘N’ must be a constant. ‘N’ must be a strict postive integer no greater than the size of the tuple. There is no cost to execute the function. The function implements the operator `x.N`. diff --git a/docs/en/sql-reference/operators/in.md b/docs/en/sql-reference/operators/in.md index 3d8d2673468..d8468370f3e 100644 --- a/docs/en/sql-reference/operators/in.md +++ b/docs/en/sql-reference/operators/in.md @@ -216,6 +216,17 @@ This is more optimal than using the normal IN. However, keep the following point It also makes sense to specify a local table in the `GLOBAL IN` clause, in case this local table is only available on the requestor server and you want to use data from it on remote servers. +### Distributed Subqueries and max_rows_in_set + +You can use [`max_rows_in_set`](../../operations/settings/query-complexity.md#max-rows-in-set) and [`max_bytes_in_set`](../../operations/settings/query-complexity.md#max-rows-in-set) to control how much data is tranferred during distributed queries. + +This is specially important if the `global in` query returns a large amount of data. Consider the following sql - +```sql +select * from table1 where col1 global in (select col1 from table2 where ) +``` + +If `some_predicate` is not selective enough, it will return large amount of data and cause performance issues. In such cases, it is wise to limit the data transfer over the network. Also, note that [`set_overflow_mode`](../../operations/settings/query-complexity.md#set_overflow_mode) is set to `throw` (by default) meaning that an exception is raised when these thresholds are met. + ### Distributed Subqueries and max_parallel_replicas {#max_parallel_replica-subqueries} When max_parallel_replicas is greater than 1, distributed queries are further transformed. For example, the following: diff --git a/docs/en/sql-reference/statements/alter/column.md b/docs/en/sql-reference/statements/alter/column.md index 2e562e20467..6bb63ea06a6 100644 --- a/docs/en/sql-reference/statements/alter/column.md +++ b/docs/en/sql-reference/statements/alter/column.md @@ -197,12 +197,13 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; ## MATERIALIZE COLUMN {#materialize-column} -Materializes the column in the parts where the column is missing. This is useful in case of creating a new column with complicated `DEFAULT` or `MATERIALIZED` expression. Calculation of the column directly on `SELECT` query can cause bigger request execution time, so it is reasonable to use `MATERIALIZE COLUMN` for such columns. To perform same manipulation for existing column, use `FINAL` modifier. +Materializes or updates a column with an expression for a default value (`DEFAULT` or `MATERIALIZED`). +It is used if it is necessary to add or update a column with a complicated expression, because evaluating such an expression directly on `SELECT` executing turns out to be expensive. Syntax: ```sql -ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; +ALTER TABLE table MATERIALIZE COLUMN col; ``` **Example** @@ -211,20 +212,34 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; DROP TABLE IF EXISTS tmp; SET mutations_sync = 2; CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple(); -INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10; +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5; ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); ALTER TABLE tmp MATERIALIZE COLUMN s; +SELECT groupArray(x), groupArray(s) FROM (select x,s from tmp order by x); + +┌─groupArray(x)─┬─groupArray(s)─────────┐ +│ [0,1,2,3,4] │ ['0','1','2','3','4'] │ +└───────────────┴───────────────────────┘ + +ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(round(100/x)); + +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5,5; + SELECT groupArray(x), groupArray(s) FROM tmp; -``` -**Result:** +┌─groupArray(x)─────────┬─groupArray(s)──────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','20','17','14','12','11'] │ +└───────────────────────┴────────────────────────────────────────────────┘ -```sql -┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐ -│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │ -└───────────────────────┴───────────────────────────────────────────┘ +ALTER TABLE tmp MATERIALIZE COLUMN s; + +SELECT groupArray(x), groupArray(s) FROM tmp; + +┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['inf','100','50','33','25','20','17','14','12','11'] │ +└───────────────────────┴───────────────────────────────────────────────────────┘ ``` **See Also** diff --git a/docs/en/sql-reference/statements/grant.md b/docs/en/sql-reference/statements/grant.md index 2b1262f7d3c..1b2b63ba0e7 100644 --- a/docs/en/sql-reference/statements/grant.md +++ b/docs/en/sql-reference/statements/grant.md @@ -172,6 +172,7 @@ Hierarchy of privileges: - `SYSTEM FLUSH LOGS` - [INTROSPECTION](#grant-introspection) - `addressToLine` + - `addressToLineWithInlines` - `addressToSymbol` - `demangle` - [SOURCES](#grant-sources) @@ -430,6 +431,7 @@ Allows using [introspection](../../operations/optimizing-performance/sampling-qu - `INTROSPECTION`. Level: `GROUP`. Aliases: `INTROSPECTION FUNCTIONS` - `addressToLine`. Level: `GLOBAL` + - `addressToLineWithInlines`. Level: `GLOBAL` - `addressToSymbol`. Level: `GLOBAL` - `demangle`. Level: `GLOBAL` diff --git a/docs/en/sql-reference/statements/select/order-by.md b/docs/en/sql-reference/statements/select/order-by.md index ee6893812cc..b24f0213e4e 100644 --- a/docs/en/sql-reference/statements/select/order-by.md +++ b/docs/en/sql-reference/statements/select/order-by.md @@ -285,7 +285,7 @@ ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_ `WITH FILL` can be applied for fields with Numeric (all kinds of float, decimal, int) or Date/DateTime types. When applied for `String` fields, missed values are filled with empty strings. When `FROM const_expr` not defined sequence of filling use minimal `expr` field value from `ORDER BY`. When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`. -When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types as `days` for Date type and as `seconds` for DateTime type. +When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types, as `days` for Date type, as `seconds` for DateTime type. It also supports [INTERVAL](https://clickhouse.com/docs/en/sql-reference/data-types/special-data-types/interval/) data type representing time and date intervals. When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type. Example of a query without `WITH FILL`: @@ -402,4 +402,85 @@ Result: └────────────┴────────────┴──────────┘ ``` +The following query uses the `INTERVAL` data type of 1 day for each data filled on column `d1`: + +``` sql +SELECT + toDate((number * 10) * 86400) AS d1, + toDate(number * 86400) AS d2, + 'original' AS source +FROM numbers(10) +WHERE (number % 3) = 1 +ORDER BY + d1 WITH FILL STEP INTERVAL 1 DAY, + d2 WITH FILL; +``` + +Result: +``` +┌─────────d1─┬─────────d2─┬─source───┐ +│ 1970-01-11 │ 1970-01-02 │ original │ +│ 1970-01-12 │ 1970-01-01 │ │ +│ 1970-01-13 │ 1970-01-01 │ │ +│ 1970-01-14 │ 1970-01-01 │ │ +│ 1970-01-15 │ 1970-01-01 │ │ +│ 1970-01-16 │ 1970-01-01 │ │ +│ 1970-01-17 │ 1970-01-01 │ │ +│ 1970-01-18 │ 1970-01-01 │ │ +│ 1970-01-19 │ 1970-01-01 │ │ +│ 1970-01-20 │ 1970-01-01 │ │ +│ 1970-01-21 │ 1970-01-01 │ │ +│ 1970-01-22 │ 1970-01-01 │ │ +│ 1970-01-23 │ 1970-01-01 │ │ +│ 1970-01-24 │ 1970-01-01 │ │ +│ 1970-01-25 │ 1970-01-01 │ │ +│ 1970-01-26 │ 1970-01-01 │ │ +│ 1970-01-27 │ 1970-01-01 │ │ +│ 1970-01-28 │ 1970-01-01 │ │ +│ 1970-01-29 │ 1970-01-01 │ │ +│ 1970-01-30 │ 1970-01-01 │ │ +│ 1970-01-31 │ 1970-01-01 │ │ +│ 1970-02-01 │ 1970-01-01 │ │ +│ 1970-02-02 │ 1970-01-01 │ │ +│ 1970-02-03 │ 1970-01-01 │ │ +│ 1970-02-04 │ 1970-01-01 │ │ +│ 1970-02-05 │ 1970-01-01 │ │ +│ 1970-02-06 │ 1970-01-01 │ │ +│ 1970-02-07 │ 1970-01-01 │ │ +│ 1970-02-08 │ 1970-01-01 │ │ +│ 1970-02-09 │ 1970-01-01 │ │ +│ 1970-02-10 │ 1970-01-05 │ original │ +│ 1970-02-11 │ 1970-01-01 │ │ +│ 1970-02-12 │ 1970-01-01 │ │ +│ 1970-02-13 │ 1970-01-01 │ │ +│ 1970-02-14 │ 1970-01-01 │ │ +│ 1970-02-15 │ 1970-01-01 │ │ +│ 1970-02-16 │ 1970-01-01 │ │ +│ 1970-02-17 │ 1970-01-01 │ │ +│ 1970-02-18 │ 1970-01-01 │ │ +│ 1970-02-19 │ 1970-01-01 │ │ +│ 1970-02-20 │ 1970-01-01 │ │ +│ 1970-02-21 │ 1970-01-01 │ │ +│ 1970-02-22 │ 1970-01-01 │ │ +│ 1970-02-23 │ 1970-01-01 │ │ +│ 1970-02-24 │ 1970-01-01 │ │ +│ 1970-02-25 │ 1970-01-01 │ │ +│ 1970-02-26 │ 1970-01-01 │ │ +│ 1970-02-27 │ 1970-01-01 │ │ +│ 1970-02-28 │ 1970-01-01 │ │ +│ 1970-03-01 │ 1970-01-01 │ │ +│ 1970-03-02 │ 1970-01-01 │ │ +│ 1970-03-03 │ 1970-01-01 │ │ +│ 1970-03-04 │ 1970-01-01 │ │ +│ 1970-03-05 │ 1970-01-01 │ │ +│ 1970-03-06 │ 1970-01-01 │ │ +│ 1970-03-07 │ 1970-01-01 │ │ +│ 1970-03-08 │ 1970-01-01 │ │ +│ 1970-03-09 │ 1970-01-01 │ │ +│ 1970-03-10 │ 1970-01-01 │ │ +│ 1970-03-11 │ 1970-01-01 │ │ +│ 1970-03-12 │ 1970-01-08 │ original │ +└────────────┴────────────┴──────────┘ +``` + [Original article](https://clickhouse.com/docs/en/sql-reference/statements/select/order-by/) diff --git a/docs/en/sql-reference/statements/system.md b/docs/en/sql-reference/statements/system.md index 23d57c22586..b71853f29dd 100644 --- a/docs/en/sql-reference/statements/system.md +++ b/docs/en/sql-reference/statements/system.md @@ -72,7 +72,7 @@ Reloads all [CatBoost](../../guides/apply-catboost-model.md#applying-catboost-mo **Syntax** ```sql -SYSTEM RELOAD MODELS +SYSTEM RELOAD MODELS [ON CLUSTER cluster_name] ``` ## RELOAD MODEL {#query_language-system-reload-model} @@ -82,7 +82,7 @@ Completely reloads a CatBoost model `model_name` if the configuration was update **Syntax** ```sql -SYSTEM RELOAD MODEL +SYSTEM RELOAD MODEL [ON CLUSTER cluster_name] ``` ## RELOAD FUNCTIONS {#query_language-system-reload-functions} @@ -92,8 +92,8 @@ Reloads all registered [executable user defined functions](../functions/index.md **Syntax** ```sql -RELOAD FUNCTIONS -RELOAD FUNCTION function_name +RELOAD FUNCTIONS [ON CLUSTER cluster_name] +RELOAD FUNCTION [ON CLUSTER cluster_name] function_name ``` ## DROP DNS CACHE {#query_language-system-drop-dns-cache} diff --git a/docs/en/sql-reference/statements/use.md b/docs/en/sql-reference/statements/use.md index 41cba58bb9d..841c23d333d 100644 --- a/docs/en/sql-reference/statements/use.md +++ b/docs/en/sql-reference/statements/use.md @@ -3,14 +3,14 @@ toc_priority: 53 toc_title: USE --- -# USE 语句 {#use} +# USE Statement {#use} ``` sql USE db ``` -用于设置会话的当前数据库。 +Lets you set the current database for the session. -如果查询语句中没有在表名前面以加点的方式指明数据库名， 则用当前数据库进行搜索。 +The current database is used for searching for tables if the database is not explicitly defined in the query with a dot before the table name. -使用 HTTP 协议时无法进行此查询，因为没有会话的概念。 +This query can’t be made when using the HTTP protocol, since there is no concept of a session. diff --git a/docs/en/sql-reference/syntax.md b/docs/en/sql-reference/syntax.md index 207b2b82cd2..19efef3dc6a 100644 --- a/docs/en/sql-reference/syntax.md +++ b/docs/en/sql-reference/syntax.md @@ -30,7 +30,7 @@ There may be any number of space symbols between syntactical constructions (incl ClickHouse supports either SQL-style and C-style comments: -- SQL-style comments start with `--` and continue to the end of the line, a space after `--` can be omitted. +- SQL-style comments start with `--`, `#!` or `# ` and continue to the end of the line, a space after `--` and `#!` can be omitted. - C-style are from `/*` to `*/`and can be multiline, spaces are not required either. ## Keywords {#syntax-keywords} @@ -106,9 +106,9 @@ In queries, you can check `NULL` using the [IS NULL](../sql-reference/operators/ ### Heredoc {#heredeoc} -A [heredoc](https://en.wikipedia.org/wiki/Here_document) is a way to define a string (often multiline), while maintaining the original formatting. A heredoc is defined as a custom string literal, placed between two `$` symbols, for example `$heredoc$`. A value between two heredocs is processed "as-is". +A [heredoc](https://en.wikipedia.org/wiki/Here_document) is a way to define a string (often multiline), while maintaining the original formatting. A heredoc is defined as a custom string literal, placed between two `$` symbols, for example `$heredoc$`. A value between two heredocs is processed "as-is". -You can use a heredoc to embed snippets of SQL, HTML, or XML code, etc. +You can use a heredoc to embed snippets of SQL, HTML, or XML code, etc. **Example** diff --git a/docs/en/whats-new/roadmap.md b/docs/en/whats-new/roadmap.md index 8872c42818f..54f8f9d68a3 100644 --- a/docs/en/whats-new/roadmap.md +++ b/docs/en/whats-new/roadmap.md @@ -5,6 +5,6 @@ toc_title: Roadmap # Roadmap {#roadmap} -The roadmap for the year 2021 is published for open discussion [here](https://github.com/ClickHouse/ClickHouse/issues/17623). +The roadmap for the year 2022 is published for open discussion [here](https://github.com/ClickHouse/ClickHouse/issues/32513). {## [Original article](https://clickhouse.com/docs/en/roadmap/) ##} diff --git a/docs/ko/images/column-oriented.gif b/docs/ko/images/column-oriented.gif new file mode 100644 index 00000000000..d5ac7c82848 Binary files /dev/null and b/docs/ko/images/column-oriented.gif differ diff --git a/docs/ko/images/logo.svg b/docs/ko/images/logo.svg new file mode 100644 index 00000000000..b5ab923ff65 --- /dev/null +++ b/docs/ko/images/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/docs/ko/images/play.png b/docs/ko/images/play.png new file mode 100644 index 00000000000..b75aebe4089 Binary files /dev/null and b/docs/ko/images/play.png differ diff --git a/docs/ko/images/row-oriented.gif b/docs/ko/images/row-oriented.gif new file mode 100644 index 00000000000..41395b5693e Binary files /dev/null and b/docs/ko/images/row-oriented.gif differ diff --git a/docs/ko/index.md b/docs/ko/index.md new file mode 100644 index 00000000000..f2a6396c069 --- /dev/null +++ b/docs/ko/index.md @@ -0,0 +1,94 @@ +--- +toc_priority: 0 +toc_title: 목차 +--- + +# ClickHouse란? {#what-is-clickhouse} + +ClickHouse® 는 query의 온라인 분석 처리(OLAP)를 위한 열 지향(column-oriented) 데이터베이스 관리 시스템(DBMS)입니다. + +"보통의" 행 지향(row-oriented) DMBS에서는 데이터가 다음과 같은 순서로 저장됩니다. + +| row | WatchID | JavaEnable | Title | GoodEvent | EventTime | +|-----|-------------|------------|--------------------|-----------|---------------------| +| #0 | 89354350662 | 1 | Investor Relations | 1 | 2016-05-18 05:19:20 | +| #1 | 90329509958 | 0 | Contact us | 1 | 2016-05-18 08:10:20 | +| #2 | 89953706054 | 1 | Mission | 1 | 2016-05-18 07:38:00 | +| #N | … | … | … | … | … | + +즉, 행과 관련된 모든 값들은 물리적으로 나란히 저장됩니다. + +행 지향(row-oriented) DMBS의 예시로는 MySQL, Postgres, 그리고 MS SQL 서버 등이 있습니다. + +열 지향 (column-oriented) DBMS에서는 데이터가 아래와 같은 방식으로 저장됩니다: + +| Row: | #0 | #1 | #2 | #N | +|-------------|---------------------|---------------------|---------------------|-----| +| WatchID: | 89354350662 | 90329509958 | 89953706054 | … | +| JavaEnable: | 1 | 0 | 1 | … | +| Title: | Investor Relations | Contact us | Mission | … | +| GoodEvent: | 1 | 1 | 1 | … | +| EventTime: | 2016-05-18 05:19:20 | 2016-05-18 08:10:20 | 2016-05-18 07:38:00 | … | + +이 예에서는 데이터가 정렬된 순서만을 보여줍니다. 다른 열의 값들은 서로 분리되어 저장되고, 같은 열의 정보들은 함께 저장됩니다. + +열 지향(column-oriented) DBMS 의 종류는 Vertica, Paraccel (Actian Matrix and Amazon Redshift), Sybase IQ, Exasol, Infobright, InfiniDB, MonetDB (VectorWise and Actian Vector), LucidDB, SAP HANA, Google Dremel, Google PowerDrill, Druid, 그리고 kdb+ 등이 있습니다. + +데이터를 저장하기 위한 서로 다른 순서는 다른 시나리오에 더 적합합니다. 데이터 접근 시나리오는 쿼리가 수행되는 빈도, 비율 및 비율을 나타내거나, 각 쿼리 유형(행, 열 및 바이트)에 대해 읽은 데이터의 양 데이터 읽기와 업데이트 사이의 관계, 데이터의 작업 크기 및 로컬에서 사용되는 방법 트랜잭션이 사용되는지 여부, 트랜잭션이 얼마나 격리되어 있는지, 데이터 복제 및 논리적 무결성에 대한 요구 사항, 각 쿼리 유형에 대한 대기 시간 및 처리량 요구 사항 등이 있습니다. + +시스템의 부하가 높을수록 사용 시나리오의 요구 사항에 맞게 시스템 설정을 사용자 지정하는 것이 더 중요하며 이 사용자 지정은 더욱 세분화됩니다. 상당히 다른 시나리오에 똑같이 적합한 시스템은 없습니다. 만약 높은 부하에서 시스템이 넓은 시나리오 집합에 대해 적응한다면 시스템은 모든 시나리오를 모두 제대로 처리하지 못하거나 가능한 시나리오 중 하나 또는 몇 개에 대해서만 잘 작동할 것입니다. + +## OLAP 시나리오의 중요 속성들 {#key-properties-of-olap-scenario} + +- 요청(request)의 대부분은 읽기 접근에 관한 것입니다. +- 데이터는 단일 행이 아니라 상당히 큰 일괄 처리(\> 1000개 행)로 업데이트됩니다. 또는 전혀 업데이트되지 않습니다. +- 데이터는 DB에 추가되지만 수정되지는 않습니다. +- 읽기의 경우 DB에서 상당히 많은 수의 행이 추출되지만 열은 일부만 추출됩니다. +- 테이블은 "넓습니다". 이는 열의 수가 많다는 것을 의미합니다. +- 쿼리는 상대적으로 드뭅니다(일반적으로 서버당 수백 또는 초당 쿼리 미만). +- 간단한 쿼리의 경우 약 50ms의 대기 시간이 허용됩니다. +- 열 값은 숫자와 짧은 문자열(예: URL당 60바이트)과 같이 상당히 작습니다 +- 단일 쿼리를 처리할 때 높은 처리량이 필요합니다(서버당 초당 최대 수십억 행). +- 트랜잭션이 필요하지 않습니다. +- 데이터 일관성에 대한 요구 사항이 낮습니다. +- 쿼리당 하나의 큰 테이블이 존재하고 하나를 제외한 모든 테이블은 작습니다. +- 쿼리 결과가 원본 데이터보다 훨씬 작습니다. 즉, 데이터가 필터링되거나 집계되므로 결과가 단일 서버의 RAM에 꼭 들어맞습니다. + +OLAP 시나리오가 다른 일반적인 시나리오(OLTP 또는 키-값 액세스와 같은)와 매우 다르다는 것을 쉽게 알 수 있습니다. 따라서 적절한 성능을 얻으려면 분석 쿼리를 처리하기 위해 OLTP 또는 키-값 DB를 사용하는 것은 의미가 없습니다. 예를 들어 분석에 MongoDB나 Redis를 사용하려고 하면 OLAP 데이터베이스에 비해 성능이 매우 저하됩니다. + +## 왜 열 지향 데이터베이스가 OLAP 시나리오에 적합한가{#why-column-oriented-databases-work-better-in-the-olap-scenario} + +열 지향(column-oriented) 데이터베이스는 OLAP 시나리오에 더 적합합니다. 대부분의 쿼리를 처리하는 데 있어서 행 지향(row-oriented) 데이터베이스보다 100배 이상 빠릅니다. 그 이유는 아래에 자세히 설명되어 있지만 사실은 시각적으로 더 쉽게 설명할 수 있습니다. + +**행 지향 DBMS** + +![Row-oriented](images/row-oriented.gif#) + +**열 지향 DBMS** + +![Column-oriented](images/column-oriented.gif#) + +차이가 보이시나요? + +### 입출력 {#inputoutput} + +1. 분석 쿼리의 경우 적은 수의 테이블 열만 읽어야 합니다. 열 지향 데이터베이스에서는 필요한 데이터만 읽을 수 있습니다. 예를 들어 100개 중 5개의 열이 필요한 경우 I/O가 20배 감소할 것으로 예상할 수 있습니다. +2. 데이터는 패킷으로 읽히므로 압축하기가 더 쉽습니다. 열의 데이터도 압축하기 쉽습니다. 이것은 I/O의 볼륨을 더욱 감소시킵니다. +3. 감소된 I/O로 인해 시스템 캐시에 더 많은 데이터가 들어갑니다. + +예를 들어, "각 광고 플랫폼에 대한 레코드 수 계산" 쿼리는 압축되지 않은 1바이트를 차지하는 하나의 "광고 플랫폼 ID" 열을 읽어야 합니다. 트래픽의 대부분이 광고 플랫폼에서 발생하지 않은 경우 이 열의 최소 10배 압축을 기대할 수 있습니다. 빠른 압축 알고리즘을 사용하면 초당 최소 몇 기가바이트의 압축되지 않은 데이터의 속도로 데이터 압축 해제가 가능합니다. 즉, 이 쿼리는 단일 서버에서 초당 약 수십억 행의 속도로 처리될 수 있습니다. 이 속도는 정말 실제로 달성됩니다. + +### CPU {#cpu} + +쿼리를 수행하려면 많은 행을 처리해야 하므로 별도의 행이 아닌 전체 벡터에 대한 모든 연산을 디스패치하거나 쿼리 엔진을 구현하여 디스패치 비용이 거의 들지 않습니다. 반쯤 괜찮은 디스크 하위 시스템에서 이렇게 하지 않으면 쿼리 인터프리터가 불가피하게 CPU를 정지시킵니다. 데이터를 열에 저장하고 가능한 경우 열별로 처리하는 것이 좋습니다. + +이를 수행하기위한 두가지 방법이 있습니다. + +1. 벡터 엔진. 모든 연산은 별도의 값 대신 벡터에 대해 작성됩니다. 즉, 작업을 자주 호출할 필요가 없으며 파견 비용도 무시할 수 있습니다. 작업 코드에는 최적화된 내부 주기가 포함되어 있습니다. +2. 코드 생성. 쿼리에 대해 생성된 코드에는 모든 간접 호출이 있습니다. + +이것은 단순한 쿼리를 실행할 때 의미가 없기 때문에 "일반" 데이터베이스에서는 수행되지 않습니다. 그러나 예외가 있습니다. 예를 들어 MemSQL은 코드 생성을 사용하여 SQL 쿼리를 처리할 때 대기 시간을 줄입니다. (비교되게, 분석 DBMS는 대기 시간이 아닌 처리량 최적화가 필요합니다.) + +CPU 효율성을 위해 쿼리 언어는 선언적(SQL 또는 MDX)이거나 최소한 벡터(J, K)여야 합니다. 쿼리는 최적화를 허용하는 암시적 루프만 포함해야 합니다. + +{## [원문](https://clickhouse.com/docs/en/) ##} diff --git a/docs/ru/development/build-osx.md b/docs/ru/development/build-osx.md index a1192b509df..48d92501f06 100644 --- a/docs/ru/development/build-osx.md +++ b/docs/ru/development/build-osx.md @@ -2,8 +2,13 @@ toc_priority: 65 toc_title: Сборка на Mac OS X --- + # Как собрать ClickHouse на Mac OS X {#how-to-build-clickhouse-on-mac-os-x} +!!! info "Вам не нужно собирать ClickHouse самостоятельно" + Вы можете установить предварительно собранный ClickHouse, как описано в [Быстром старте](https://clickhouse.com/#quick-start). + Следуйте инструкциям по установке для `macOS (Intel)` или `macOS (Apple Silicon)`. + Сборка должна запускаться с x86_64 (Intel) на macOS версии 10.15 (Catalina) и выше в последней версии компилятора Xcode's native AppleClang, Homebrew's vanilla Clang или в GCC-компиляторах. ## Установка Homebrew {#install-homebrew} diff --git a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md index 9cd8eda0b87..117223127c0 100644 --- a/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/graphitemergetree.md @@ -99,13 +99,16 @@ patterns ``` text pattern + rule_type regexp function pattern + rule_type regexp age + precision ... pattern + rule_type regexp function age + precision @@ -129,12 +132,20 @@ default Поля для разделов `pattern` и `default`: -- `regexp` – шаблон имени метрики. +- `rule_type` - тип правила (применяется только к метрикам указанных типов), используется для разделения правил проверки плоских/теггированных метрик. Опциональное поле. Значение по умолчанию: `all`. +Если используются метрики только одного типа или производительность проверки правил некритична, можно не использовать. По умолчанию создается только один тип правил для проверки. Иначе, если хотя бы для одного правила указано отличное от умолчания значение, создаются 2 независимых типа правил - для обычных (классические root.branch.leaf) и теггированных метрик (root.branch.leaf;tag1=value1). +Правила по умолчанию попадают в оба правила обоих типов. +Возможные значения: + - `all` (default) - универсальное правило, назначается также по умолчанию, если поле не задано + - `plain` - правило для плоских метрик (без тегов). Поле `regexp` обрабатывается как регулярное выражение. + - `tagged` - правило для теггированных метрик (метрика хранится в БД в формате `someName?tag1=value1&tag2=value2&tag3=value3`), регулярное выражение должно быть отсортированно по именам тегов, первым - значение тега `__name__`, если есть. Поле `regexp` обрабатывается как регулярное выражение. + - `tag_list` - правило для теггированных метрик, простой DSL для упрощения задания регулярного выражения в формате тегов graphite `someName;tag1=value1;tag2=value2`, `someName` или `tag1=value1;tag2=value2`. Поле `regexp` транслируется в правило `tagged`. Cортировать по именам тегов не обязательно, оно отсортируется автоматически. Значение тега (но не имя) может быть регулярным выражением (например `env=(dev|staging)`). +- `regexp` – шаблон имени метрики (регулярное выражение или DSL). - `age` – минимальный возраст данных в секундах. - `precision` – точность определения возраста данных в секундах. Должен быть делителем для 86400 (количество секунд в сутках). - `function` – имя агрегирующей функции, которую следует применить к данным, чей возраст оказался в интервале `[age, age + precision]`. Допустимые функции: min/max/any/avg. Avg вычисляется неточно, как среднее от средних. -### Пример конфигурации {#configuration-example} +### Пример конфигурации без разделения типа правил {#configuration-example} ``` xml @@ -169,6 +180,80 @@ default ``` +### Пример конфигурации c разделением типа правил {#configuration-typed-example} + +``` xml + + Version + + plain + click_cost + any + + 0 + 5 + + + 86400 + 60 + + + + tagged + ^((.*)|.)min\? + min + + 0 + 5 + + + 86400 + 60 + + + + tagged + + min + + 0 + 5 + + + 86400 + 60 + + + + tag_list + someName;tag2=value2 + + 0 + 5 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + +``` + !!! warning "Внимание" Прореживание данных производится во время слияний. Обычно для старых партиций слияния не запускаются, поэтому для прореживания надо инициировать незапланированное слияние используя [optimize](../../../sql-reference/statements/optimize.md). Или использовать дополнительные инструменты, например [graphite-ch-optimizer](https://github.com/innogames/graphite-ch-optimizer). diff --git a/docs/ru/engines/table-engines/mergetree-family/mergetree.md b/docs/ru/engines/table-engines/mergetree-family/mergetree.md index 4448372c522..3f140f85396 100644 --- a/docs/ru/engines/table-engines/mergetree-family/mergetree.md +++ b/docs/ru/engines/table-engines/mergetree-family/mergetree.md @@ -872,3 +872,13 @@ SETTINGS storage_policy = 'moving_from_ssd_to_hdd' ``` Если диск сконфигурирован как `cold`, данные будут переноситься в S3 при срабатывании правил TTL или когда свободное место на локальном диске станет меньше порогового значения, которое определяется как `move_factor * disk_size`. + +## Виртуальные столбцы {#virtual-columns} + +- `_part` — Имя куска. +- `_part_index` — Номер куска по порядку в результате запроса. +- `_partition_id` — Имя партиции. +- `_part_uuid` — Уникальный идентификатор куска (если включена MergeTree настройка `assign_part_uuids`). +- `_partition_value` — Значения (кортеж) выражения `partition by`. +- `_sample_factor` — Коэффициент сэмплирования (из запроса). + diff --git a/docs/ru/engines/table-engines/special/buffer.md b/docs/ru/engines/table-engines/special/buffer.md index 0c1ae591ae3..10b4b9645a2 100644 --- a/docs/ru/engines/table-engines/special/buffer.md +++ b/docs/ru/engines/table-engines/special/buffer.md @@ -48,10 +48,8 @@ CREATE TABLE merge.hits_buffer AS merge.hits ENGINE = Buffer(merge, hits, 16, 10 Если у одного из столбцов таблицы Buffer и подчинённой таблицы не совпадает тип, то в лог сервера будет записано сообщение об ошибке и буфер будет очищен. То же самое происходит, если подчинённая таблица не существует в момент сброса буфера. -Если есть необходимость выполнить ALTER для подчинённой таблицы и для таблицы Buffer, то рекомендуется удалить таблицу Buffer, затем выполнить ALTER подчинённой таблицы, а после создать таблицу Buffer заново. - !!! attention "Внимание" - В релизах до 28 сентября 2020 года выполнение ALTER на таблице Buffer ломает структуру блоков и вызывает ошибку (см. [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117)), поэтому удаление буфера и его пересоздание — единственный вариант миграции для данного движка. Перед выполнением ALTER на таблице Buffer убедитесь, что в вашей версии эта ошибка устранена. + В релизах до 26 октября 2021 года выполнение ALTER на таблице Buffer ломает структуру блоков и вызывает ошибку (см. [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117) и [#30565](https://github.com/ClickHouse/ClickHouse/pull/30565)), поэтому удаление буфера и его пересоздание — единственный вариант миграции для данного движка. Перед выполнением ALTER на таблице Buffer убедитесь, что в вашей версии эта ошибка устранена. При нештатном перезапуске сервера, данные, находящиеся в буфере, будут потеряны. diff --git a/docs/ru/operations/settings/settings.md b/docs/ru/operations/settings/settings.md index affa90d9840..5e4b7c6bcb7 100644 --- a/docs/ru/operations/settings/settings.md +++ b/docs/ru/operations/settings/settings.md @@ -1736,6 +1736,48 @@ ClickHouse генерирует исключение: Т.е. если `INSERT` в основную таблицу д.б. пропущен (сдедуплицирован), то автоматически не будет вставки и в материализованные представления. Это имплементировано для того, чтобы работали материализованные представления, которые сильно группируют данные основных `INSERT`, до такой степени что блоки вставляемые в материализованные представления получаются одинаковыми для разных `INSERT` в основную таблицу. Одновременно это «ломает» идемпотентность вставки в материализованные представления. Т.е. если `INSERT` был успешен в основную таблицу и неуспешен в таблицу материализованного представления (напр. из-за сетевого сбоя при коммуникации с Zookeeper), клиент получит ошибку и попытается повторить `INSERT`. Но вставки в материализованные представления произведено не будет, потому что дедупликация сработает на основной таблице. Настройка `deduplicate_blocks_in_dependent_materialized_views` позволяет это изменить. Т.е. при повторном `INSERT` будет произведена дедупликация на таблице материализованного представления, и повторный инсерт вставит данные в таблицу материализованного представления, которые не удалось вставить из-за сбоя первого `INSERT`. +## insert_deduplication_token {#insert_deduplication_token} + +Этот параметр позволяет пользователю указать собственную семантику дедупликации в MergeTree/ReplicatedMergeTree. +Например, предоставляя уникальное значение параметра в каждом операторе INSERT, +пользователь может избежать дедупликации одних и тех же вставленных данных. + +Возможные значения: + +- Любая строка + +Значение по умолчанию: пустая строка (выключено). + +`insert_deduplication_token` используется для дедупликации _только_ когда значение не пустое + +Example: + +```sql +CREATE TABLE test_table +( A Int64 ) +ENGINE = MergeTree +ORDER BY A +SETTINGS non_replicated_deduplication_window = 100; + +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test' (1); + +-- следующая вставка не будет дедуплицирована, потому что insert_deduplication_token отличается +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test1' (1); + +-- следующая вставка будет дедуплицирована, потому что insert_deduplication_token +-- тот же самый, что и один из предыдущих +INSERT INTO test_table Values SETTINGS insert_deduplication_token = 'test' (2); + +SELECT * FROM test_table + +┌─A─┐ +│ 1 │ +└───┘ +┌─A─┐ +│ 1 │ +└───┘ +``` + ## count_distinct_implementation {#settings-count_distinct_implementation} Задаёт, какая из функций `uniq*` используется при выполнении конструкции [COUNT(DISTINCT …)](../../sql-reference/aggregate-functions/reference/count.md#agg_function-count). @@ -2119,7 +2161,7 @@ ClickHouse генерирует исключение: - 1 — включен режим параллельного разбора. - 0 — отключен режим параллельного разбора. -Значение по умолчанию: `0`. +Значение по умолчанию: `1`. ## output_format_parallel_formatting {#output-format-parallel-formatting} @@ -2130,7 +2172,7 @@ ClickHouse генерирует исключение: - 1 — включен режим параллельного форматирования. - 0 — отключен режим параллельного форматирования. -Значение по умолчанию: `0`. +Значение по умолчанию: `1`. ## min_chunk_bytes_for_parallel_parsing {#min-chunk-bytes-for-parallel-parsing} diff --git a/docs/ru/sql-reference/statements/alter/column.md b/docs/ru/sql-reference/statements/alter/column.md index 4de2d067cce..fea4c00ac05 100644 --- a/docs/ru/sql-reference/statements/alter/column.md +++ b/docs/ru/sql-reference/statements/alter/column.md @@ -197,12 +197,13 @@ ALTER TABLE table_with_ttl MODIFY COLUMN column_ttl REMOVE TTL; ## MATERIALIZE COLUMN {#materialize-column} -Материализует столбец таблицы в кусках, в которых отсутствуют значения. Используется, если необходимо создать новый столбец со сложным материализованным выражением или выражением для заполнения по умолчанию (`DEFAULT`), потому как вычисление такого столбца прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным. Чтобы совершить ту же операцию для существующего столбца, используйте модификатор `FINAL`. +Материализует или обновляет столбец таблицы с выражением для значения по умолчанию (`DEFAULT` или `MATERIALIZED`). +Используется, если необходимо добавить или обновить столбец со сложным выражением, потому как вычисление такого выражения прямо во время выполнения запроса `SELECT` оказывается ощутимо затратным. Синтаксис: ```sql -ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; +ALTER TABLE table MATERIALIZE COLUMN col; ``` **Пример** @@ -211,21 +212,39 @@ ALTER TABLE table MATERIALIZE COLUMN col [FINAL]; DROP TABLE IF EXISTS tmp; SET mutations_sync = 2; CREATE TABLE tmp (x Int64) ENGINE = MergeTree() ORDER BY tuple() PARTITION BY tuple(); -INSERT INTO tmp SELECT * FROM system.numbers LIMIT 10; +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5; ALTER TABLE tmp ADD COLUMN s String MATERIALIZED toString(x); ALTER TABLE tmp MATERIALIZE COLUMN s; +SELECT groupArray(x), groupArray(s) FROM (select x,s from tmp order by x); + +┌─groupArray(x)─┬─groupArray(s)─────────┐ +│ [0,1,2,3,4] │ ['0','1','2','3','4'] │ +└───────────────┴───────────────────────┘ + +ALTER TABLE tmp MODIFY COLUMN s String MATERIALIZED toString(round(100/x)); + +INSERT INTO tmp SELECT * FROM system.numbers LIMIT 5,5; + SELECT groupArray(x), groupArray(s) FROM tmp; + +┌─groupArray(x)─────────┬─groupArray(s)──────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','20','17','14','12','11'] │ +└───────────────────────┴────────────────────────────────────────────────┘ + +ALTER TABLE tmp MATERIALIZE COLUMN s; + +SELECT groupArray(x), groupArray(s) FROM tmp; + +┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────────────────┐ +│ [0,1,2,3,4,5,6,7,8,9] │ ['inf','100','50','33','25','20','17','14','12','11'] │ +└───────────────────────┴───────────────────────────────────────────────────────┘ ``` -**Результат:** +**Смотрите также** -```sql -┌─groupArray(x)─────────┬─groupArray(s)─────────────────────────────┐ -│ [0,1,2,3,4,5,6,7,8,9] │ ['0','1','2','3','4','5','6','7','8','9'] │ -└───────────────────────┴───────────────────────────────────────────┘ -``` +- [MATERIALIZED](../../statements/create/table.md#materialized). ## Ограничения запроса ALTER {#ogranicheniia-zaprosa-alter} diff --git a/docs/ru/sql-reference/syntax.md b/docs/ru/sql-reference/syntax.md index 6705b1068fe..7e9260915a8 100644 --- a/docs/ru/sql-reference/syntax.md +++ b/docs/ru/sql-reference/syntax.md @@ -28,7 +28,7 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ## Комментарии {#comments} Поддерживаются комментарии в SQL-стиле и C-стиле. -Комментарии в SQL-стиле: от `--` до конца строки. Пробел после `--` может не ставиться. +Комментарии в SQL-стиле: от `--`, `#!` или `# ` до конца строки. Пробел после `--` и `#!` может не ставиться. Комментарии в C-стиле: от `/*` до `*/`. Такие комментарии могут быть многострочными. Пробелы тоже не обязательны. ## Ключевые слова {#syntax-keywords} @@ -104,9 +104,9 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def') ### Heredoc {#heredeoc} -Синтаксис [heredoc](https://ru.wikipedia.org/wiki/Heredoc-синтаксис) — это способ определения строк с сохранением исходного формата (часто с переносом строки). `Heredoc` задается как произвольный строковый литерал между двумя символами `$`, например `$heredoc$`. Значение между двумя `heredoc` обрабатывается "как есть". +Синтаксис [heredoc](https://ru.wikipedia.org/wiki/Heredoc-синтаксис) — это способ определения строк с сохранением исходного формата (часто с переносом строки). `Heredoc` задается как произвольный строковый литерал между двумя символами `$`, например `$heredoc$`. Значение между двумя `heredoc` обрабатывается "как есть". -Синтаксис `heredoc` часто используют для вставки кусков кода SQL, HTML, XML и т.п. +Синтаксис `heredoc` часто используют для вставки кусков кода SQL, HTML, XML и т.п. **Пример** diff --git a/docs/tools/single_page.py b/docs/tools/single_page.py index cf41e2b78c2..3d32ba30a21 100644 --- a/docs/tools/single_page.py +++ b/docs/tools/single_page.py @@ -90,7 +90,10 @@ def concatenate(lang, docs_path, single_page_file, nav): line) # If failed to replace the relative link, print to log - if '../' in line: + # But with some exceptions: + # - "../src/" -- for cmake-in-clickhouse.md (link to sources) + # - "../usr/share" -- changelog entry that has "../usr/share/zoneinfo" + if '../' in line and (not '../usr/share' in line) and (not '../src/' in line): logging.info('Failed to resolve relative link:') logging.info(path) logging.info(line) diff --git a/docs/zh/engines/database-engines/index.md b/docs/zh/engines/database-engines/index.md index 10be2e0f041..0d844365fbb 100644 --- a/docs/zh/engines/database-engines/index.md +++ b/docs/zh/engines/database-engines/index.md @@ -14,7 +14,7 @@ toc_title: Introduction - [MySQL](../../engines/database-engines/mysql.md) -- [MaterializeMySQL](../../engines/database-engines/materialize-mysql.md) +- [MaterializeMySQL](../../engines/database-engines/materialized-mysql.md) - [Lazy](../../engines/database-engines/lazy.md) @@ -26,4 +26,6 @@ toc_title: Introduction - [Replicated](../../engines/database-engines/replicated.md) +- [SQLite](../../engines/database-engines/sqlite.md) + [来源文章](https://clickhouse.com/docs/en/database_engines/) diff --git a/docs/zh/engines/database-engines/materialized-mysql.md b/docs/zh/engines/database-engines/materialized-mysql.md deleted file mode 120000 index 02118b85df4..00000000000 --- a/docs/zh/engines/database-engines/materialized-mysql.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/engines/database-engines/materialized-mysql.md \ No newline at end of file diff --git a/docs/zh/engines/database-engines/materialized-mysql.md b/docs/zh/engines/database-engines/materialized-mysql.md new file mode 100644 index 00000000000..f654013494a --- /dev/null +++ b/docs/zh/engines/database-engines/materialized-mysql.md @@ -0,0 +1,274 @@ +--- +toc_priority: 29 +toc_title: MaterializedMySQL +--- + +# [experimental] MaterializedMySQL {#materialized-mysql} + +!!! warning "警告" + 这是一个实验性的特性，不应该在生产中使用. + + +创建ClickHouse数据库，包含MySQL中所有的表，以及这些表中的所有数据。 + +ClickHouse服务器作为MySQL副本工作。它读取binlog并执行DDL和DML查询。 + +## 创建数据库 {#creating-a-database} + +``` sql +CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster] +ENGINE = MaterializedMySQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...] +[TABLE OVERRIDE table1 (...), TABLE OVERRIDE table2 (...)] +``` + +**引擎参数** + +- `host:port` — MySQL 服务地址. +- `database` — MySQL 数据库名称. +- `user` — MySQL 用户名. +- `password` — MySQL 用户密码. + +**引擎配置** + + +- `max_rows_in_buffer` — 允许在内存中缓存数据的最大行数(对于单个表和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值:`65 505`。 +- `max_bytes_in_buffer` - 允许在内存中缓存数据的最大字节数(对于单个表和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `1 048 576 `。 +- `max_rows_in_buffers` - 允许在内存中缓存数据的最大行数(用于数据库和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `65 505`。 +- `max_bytes_in_buffers` - 允许在内存中缓存数据的最大字节数(用于数据库和无法查询的缓存数据)。当超过这个数字时，数据将被物化。默认值: `1 048 576`。 +- `max_flush_data_time ` - 允许数据在内存中缓存的最大毫秒数(对于数据库和无法查询的缓存数据)。当超过这个时间，数据将被物化。默认值: `1000`。 +- `max_wait_time_when_mysql_unavailable` - MySQL不可用时的重试间隔(毫秒)。负值禁用重试。默认值:`1000`。 +— `allows_query_when_mysql_lost `—允许在MySQL丢失时查询物化表。默认值:`0`(`false`)。 + +```sql +CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***') + SETTINGS + allows_query_when_mysql_lost=true, + max_wait_time_when_mysql_unavailable=10000; +``` + +**MySQL服务器端配置** + +为了`MaterializedMySQL`的正确工作，有一些必须设置的`MySQL`端配置设置: + +- `default_authentication_plugin = mysql_native_password `，因为 `MaterializedMySQL` 只能授权使用该方法。 +- `gtid_mode = on`，因为基于GTID的日志记录是提供正确的 `MaterializedMySQL`复制的强制要求。 + +!!! attention "注意" + 当打开`gtid_mode`时，您还应该指定`enforce_gtid_consistency = on`。 + +## 虚拟列 {#virtual-columns} + +当使用`MaterializeMySQL`数据库引擎时，[ReplacingMergeTree](../../engines/table-engines/mergetree-family/replacingmergetree.md)表与虚拟的`_sign`和`_version`列一起使用。 + +- `_version` — 事务版本. 类型 [UInt64](../../sql-reference/data-types/int-uint.md). +- `_sign` — 删除标记. 类型 [Int8](../../sql-reference/data-types/int-uint.md). 可能的值: + - `1` — 行没有删除, + - `-1` — 行已被删除. + +## 支持的数据类型 {#data_types-support} + +| MySQL | ClickHouse | +|-------------------------|--------------------------------------------------------------| +| TINY | [Int8](../../sql-reference/data-types/int-uint.md) | +| SHORT | [Int16](../../sql-reference/data-types/int-uint.md) | +| INT24 | [Int32](../../sql-reference/data-types/int-uint.md) | +| LONG | [UInt32](../../sql-reference/data-types/int-uint.md) | +| LONGLONG | [UInt64](../../sql-reference/data-types/int-uint.md) | +| FLOAT | [Float32](../../sql-reference/data-types/float.md) | +| DOUBLE | [Float64](../../sql-reference/data-types/float.md) | +| DECIMAL, NEWDECIMAL | [Decimal](../../sql-reference/data-types/decimal.md) | +| DATE, NEWDATE | [Date](../../sql-reference/data-types/date.md) | +| DATETIME, TIMESTAMP | [DateTime](../../sql-reference/data-types/datetime.md) | +| DATETIME2, TIMESTAMP2 | [DateTime64](../../sql-reference/data-types/datetime64.md) | +| YEAR | [UInt16](../../sql-reference/data-types/int-uint.md) | +| TIME | [Int64](../../sql-reference/data-types/int-uint.md) | +| ENUM | [Enum](../../sql-reference/data-types/enum.md) | +| STRING | [String](../../sql-reference/data-types/string.md) | +| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) | +| BLOB | [String](../../sql-reference/data-types/string.md) | +| GEOMETRY | [String](../../sql-reference/data-types/string.md) | +| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) | +| BIT | [UInt64](../../sql-reference/data-types/int-uint.md) | +| SET | [UInt64](../../sql-reference/data-types/int-uint.md) | + +[Nullable](../../sql-reference/data-types/nullable.md) 已经被支持. + +MySQL中的Time 类型，会被ClickHouse转换成微秒来存储 + +不支持其他类型。如果MySQL表包含此类类型的列，ClickHouse抛出异常"Unhandled data type"并停止复制。 + +## 规范和推荐用法 {#specifics-and-recommendations} + +### 兼容性限制 {#compatibility-restrictions} + +除了数据类型的限制之外，还有一些限制与`MySQL`数据库相比有所不同，这应该在复制之前解决: + +- `MySQL` 中的每个表都应该包含 `PRIMARY KEY`。 +- 对于表的复制，那些包含 `ENUM` 字段值超出范围的行(在 `ENUM` 签名中指定)将不起作用。 + +### DDL Queries {#ddl-queries} + +MySQL DDL 语句会被转换成对应的ClickHouse DDL 语句，比如： ([ALTER](../../sql-reference/statements/alter/index.md), [CREATE](../../sql-reference/statements/create/index.md), [DROP](../../sql-reference/statements/drop.md), [RENAME](../../sql-reference/statements/rename.md)). 如果ClickHouse 无法解析某些语句DDL 操作，则会跳过。 + + +### 数据复制 {#data-replication} + +MaterializedMySQL不支持直接的 `INSERT`， `DELETE` 和 `UPDATE` 查询。然而，它们在数据复制方面得到了支持: + +- MySQL `INSERT`查询被转换为`_sign=1`的INSERT查询。 +- MySQL `DELETE`查询被转换为`INSERT`，并且`_sign=-1`。 +- 如果主键被修改了，MySQL的 `UPDATE` 查询将被转换为 `INSERT` 带 `_sign=1` 和INSERT 带有_sign=-1;如果主键没有被修改，则转换为`INSERT`和`_sign=1`。 + +### MaterializedMySQL 数据表查询 {#select} + +`SELECT` 查询从 `MaterializedMySQL`表有一些细节: + + - 如果在SELECT查询中没有指定`_version`，则 [FINAL](../../sql-reference/statements/select/from.md#select-from- FINAL)修饰符被使用，所以只有带有 `MAX(_version)`的行会返回每个主键值。 + + - 如果在SELECT查询中没有指定 `_sign`，则默认使用 `WHERE _sign=1 `。所以被删除的行不是 +包含在结果集中。 + + - 结果包括列注释，以防MySQL数据库表中存在这些列注释。 + +### 索引转换 {#index-conversion} + +在ClickHouse表中，MySQL的 `PRIMARY KEY` 和 `INDEX` 子句被转换为 `ORDER BY` 元组。 + +ClickHouse只有一个物理排序，由 `order by` 条件决定。要创建一个新的物理排序，请使用[materialized views](../../sql-reference/statements/create/view.md#materialized)。 + +**注意** + +- `_sign=-1` 的行不会被物理地从表中删除。 +- 级联 `UPDATE/DELETE` 查询不支持 `MaterializedMySQL` 引擎，因为他们在 MySQL binlog中不可见的 +— 复制很容易被破坏。 +— 禁止对数据库和表进行手工操作。 +- `MaterializedMySQL` 受[optimize_on_insert](../../operations/settings/settings.md#optimize-on-insert)设置的影响。当MySQL服务器中的一个表发生变化时，数据会合并到 `MaterializedMySQL` 数据库中相应的表中。 + +### 表重写 {#table-overrides} + +表覆盖可用于自定义ClickHouse DDL查询，从而允许您对应用程序进行模式优化。这对于控制分区特别有用，分区对MaterializedMySQL的整体性能非常重要。 + +这些是你可以对MaterializedMySQL表重写的模式转换操作: + + * 修改列类型。必须与原始类型兼容，否则复制将失败。例如，可以将`UInt32`列修改为`UInt64`，不能将 `String` 列修改为 `Array(String)`。 + * 修改 [column TTL](../table-engines/mergetree-family/mergetree/#mergetree-column-ttl). + * 修改 [column compression codec](../../sql-reference/statements/create/table/#codecs). + * 增加 [ALIAS columns](../../sql-reference/statements/create/table/#alias). + * 增加 [skipping indexes](../table-engines/mergetree-family/mergetree/#table_engine-mergetree-data_skipping-indexes) + * 增加 [projections](../table-engines/mergetree-family/mergetree/#projections). + 请注意，当使用 `SELECT ... FINAL ` (MaterializedMySQL默认是这样做的) 时，预测优化是被禁用的，所以这里是受限的， `INDEX ... TYPE hypothesis `[在v21.12的博客文章中描述]](https://clickhouse.com/blog/en/2021/clickhouse-v21.12-released/)可能在这种情况下更有用。 + * 修改 [PARTITION BY](../table-engines/mergetree-family/custom-partitioning-key/) + * 修改 [ORDER BY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses) + * 修改 [PRIMARY KEY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses) + * 增加 [SAMPLE BY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses) + * 增加 [table TTL](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses) + +```sql +CREATE DATABASE db_name ENGINE = MaterializedMySQL(...) +[SETTINGS ...] +[TABLE OVERRIDE table_name ( + [COLUMNS ( + [col_name [datatype] [ALIAS expr] [CODEC(...)] [TTL expr], ...] + [INDEX index_name expr TYPE indextype[(...)] GRANULARITY val, ...] + [PROJECTION projection_name (SELECT [GROUP BY] [ORDER BY]), ...] + )] + [ORDER BY expr] + [PRIMARY KEY expr] + [PARTITION BY expr] + [SAMPLE BY expr] + [TTL expr] +), ...] +``` + +示例: + +```sql +CREATE DATABASE db_name ENGINE = MaterializedMySQL(...) +TABLE OVERRIDE table1 ( + COLUMNS ( + userid UUID, + category LowCardinality(String), + timestamp DateTime CODEC(Delta, Default) + ) + PARTITION BY toYear(timestamp) +), +TABLE OVERRIDE table2 ( + COLUMNS ( + client_ip String TTL created + INTERVAL 72 HOUR + ) + SAMPLE BY ip_hash +) +``` + + +`COLUMNS`列表是稀疏的;根据指定修改现有列，添加额外的ALIAS列。不可能添加普通列或实体化列。具有不同类型的已修改列必须可从原始类型赋值。在执行`CREATE DATABASE` 查询时，目前还没有验证这个或类似的问题，因此需要格外小心。 + +您可以为还不存在的表指定重写。 + +!!! warning "警告" + 如果使用时不小心，很容易用表重写中断复制。例如: + + * 如果一个ALIAS列被添加了一个表覆盖，并且一个具有相同名称的列后来被添加到源MySQL表，在ClickHouse中转换后的ALTER table查询将失败并停止复制。 + * 目前可以添加引用可空列的覆盖，而非空列是必需的，例如 `ORDER BY` 或 `PARTITION BY`。这将导致CREATE TABLE查询失败，也会导致复制停止。 + +## 使用示例 {#examples-of-use} + + MySQL 查询语句: + +``` sql +mysql> CREATE DATABASE db; +mysql> CREATE TABLE db.test (a INT PRIMARY KEY, b INT); +mysql> INSERT INTO db.test VALUES (1, 11), (2, 22); +mysql> DELETE FROM db.test WHERE a=1; +mysql> ALTER TABLE db.test ADD COLUMN c VARCHAR(16); +mysql> UPDATE db.test SET c='Wow!', b=222; +mysql> SELECT * FROM test; +``` + +```text +┌─a─┬───b─┬─c────┐ +│ 2 │ 222 │ Wow! │ +└───┴─────┴──────┘ +``` + +ClickHouse中的数据库，与MySQL服务器交换数据: + +创建的数据库和表: + +``` sql +CREATE DATABASE mysql ENGINE = MaterializedMySQL('localhost:3306', 'db', 'user', '***'); +SHOW TABLES FROM mysql; +``` + +``` text +┌─name─┐ +│ test │ +└──────┘ +``` + +数据插入之后： + +``` sql +SELECT * FROM mysql.test; +``` + +``` text +┌─a─┬──b─┐ +│ 1 │ 11 │ +│ 2 │ 22 │ +└───┴────┘ +``` + +删除数据后，添加列并更新: + +``` sql +SELECT * FROM mysql.test; +``` + +``` text +┌─a─┬───b─┬─c────┐ +│ 2 │ 222 │ Wow! │ +└───┴─────┴──────┘ +``` + +[来源文章](https://clickhouse.com/docs/en/engines/database-engines/materialized-mysql/) diff --git a/docs/zh/engines/database-engines/postgresql.md b/docs/zh/engines/database-engines/postgresql.md index 12b8133f404..4d2af9182f9 100644 --- a/docs/zh/engines/database-engines/postgresql.md +++ b/docs/zh/engines/database-engines/postgresql.md @@ -24,6 +24,7 @@ ENGINE = PostgreSQL('host:port', 'database', 'user', 'password'[, `use_table_cac - `database` — 远程数据库名次 - `user` — PostgreSQL用户名称 - `password` — PostgreSQL用户密码 +- `schema` - PostgreSQL 模式 - `use_table_cache` — 定义数据库表结构是否已缓存或不进行。可选的。默认值： `0`. ## 支持的数据类型 {#data_types-support} diff --git a/docs/zh/engines/database-engines/replicated.md b/docs/zh/engines/database-engines/replicated.md index 9ffebe04571..bd5841491dd 100644 --- a/docs/zh/engines/database-engines/replicated.md +++ b/docs/zh/engines/database-engines/replicated.md @@ -31,6 +31,7 @@ CREATE DATABASE testdb ENGINE = Replicated('zoo_path', 'shard_name', 'replica_na 当创建数据库的新副本时，该副本会自己创建表。如果副本已经不可用很长一段时间，并且已经滞后于复制日志-它用ZooKeeper中的当前元数据检查它的本地元数据，将带有数据的额外表移动到一个单独的非复制数据库(以免意外地删除任何多余的东西)，创建缺失的表，如果表名已经被重命名，则更新表名。数据在`ReplicatedMergeTree`级别被复制，也就是说，如果表没有被复制，数据将不会被复制(数据库只负责元数据)。 +允许[`ALTER TABLE ATTACH|FETCH|DROP|DROP DETACHED|DETACH PARTITION|PART`](../../sql-reference/statements/alter/partition.md)查询，但不允许复制。数据库引擎将只向当前副本添加/获取/删除分区/部件。但是，如果表本身使用了Replicated表引擎，那么数据将在使用`ATTACH`后被复制。 ## 使用示例 {#usage-example} 创建三台主机的集群: diff --git a/docs/zh/engines/database-engines/sqlite.md b/docs/zh/engines/database-engines/sqlite.md deleted file mode 120000 index 776734647c2..00000000000 --- a/docs/zh/engines/database-engines/sqlite.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/engines/database-engines/sqlite.md \ No newline at end of file diff --git a/docs/zh/engines/database-engines/sqlite.md b/docs/zh/engines/database-engines/sqlite.md new file mode 100644 index 00000000000..48dca38e4af --- /dev/null +++ b/docs/zh/engines/database-engines/sqlite.md @@ -0,0 +1,80 @@ +--- +toc_priority: 32 +toc_title: SQLite +--- + +# SQLite {#sqlite} + +允许连接到[SQLite](https://www.sqlite.org/index.html)数据库，并支持ClickHouse和SQLite交换数据， 执行 `INSERT` 和 `SELECT` 查询。 + +## 创建一个数据库 {#creating-a-database} + +``` sql + CREATE DATABASE sqlite_database + ENGINE = SQLite('db_path') +``` + +**引擎参数** + +- `db_path` — SQLite 数据库文件的路径. + +## 数据类型的支持 {#data_types-support} + +| SQLite | ClickHouse | +|---------------|---------------------------------------------------------| +| INTEGER | [Int32](../../sql-reference/data-types/int-uint.md) | +| REAL | [Float32](../../sql-reference/data-types/float.md) | +| TEXT | [String](../../sql-reference/data-types/string.md) | +| BLOB | [String](../../sql-reference/data-types/string.md) | + +## 技术细节和建议 {#specifics-and-recommendations} + +SQLite将整个数据库(定义、表、索引和数据本身)存储为主机上的单个跨平台文件。在写入过程中，SQLite会锁定整个数据库文件，因此写入操作是顺序执行的。读操作可以是多任务的。 +SQLite不需要服务管理(如启动脚本)或基于`GRANT`和密码的访问控制。访问控制是通过授予数据库文件本身的文件系统权限来处理的。 + +## 使用示例 {#usage-example} + +数据库在ClickHouse，连接到SQLite: + +``` sql +CREATE DATABASE sqlite_db ENGINE = SQLite('sqlite.db'); +SHOW TABLES FROM sqlite_db; +``` + +``` text +┌──name───┐ +│ table1 │ +│ table2 │ +└─────────┘ +``` + +展示数据表中的内容: + +``` sql +SELECT * FROM sqlite_db.table1; +``` + +``` text +┌─col1──┬─col2─┐ +│ line1 │ 1 │ +│ line2 │ 2 │ +│ line3 │ 3 │ +└───────┴──────┘ +``` +从ClickHouse表插入数据到SQLite表: + +``` sql +CREATE TABLE clickhouse_table(`col1` String,`col2` Int16) ENGINE = MergeTree() ORDER BY col2; +INSERT INTO clickhouse_table VALUES ('text',10); +INSERT INTO sqlite_db.table1 SELECT * FROM clickhouse_table; +SELECT * FROM sqlite_db.table1; +``` + +``` text +┌─col1──┬─col2─┐ +│ line1 │ 1 │ +│ line2 │ 2 │ +│ line3 │ 3 │ +│ text │ 10 │ +└───────┴──────┘ +``` diff --git a/docs/zh/engines/table-engines/integrations/hive.md b/docs/zh/engines/table-engines/integrations/hive.md new file mode 100644 index 00000000000..aa2c82d902a --- /dev/null +++ b/docs/zh/engines/table-engines/integrations/hive.md @@ -0,0 +1,416 @@ +--- +toc_priority: 4 +toc_title: Hive +--- + +# Hive {#hive} + +Hive引擎允许对HDFS Hive表执行 `SELECT` 查询。目前它支持如下输入格式: + +-文本:只支持简单的标量列类型，除了 `Binary` + +- ORC:支持简单的标量列类型，除了`char`; 只支持 `array` 这样的复杂类型 + +- Parquet:支持所有简单标量列类型;只支持 `array` 这样的复杂类型 + +## 创建表 {#creating-a-table} + +``` sql +CREATE TABLE [IF NOT EXISTS] [db.]table_name [ON CLUSTER cluster] +( + name1 [type1] [ALIAS expr1], + name2 [type2] [ALIAS expr2], + ... +) ENGINE = Hive('thrift://host:port', 'database', 'table'); +PARTITION BY expr +``` +查看[CREATE TABLE](../../../sql-reference/statements/create/table.md#create-table-query)查询的详细描述。 + +表的结构可以与原来的Hive表结构有所不同: +- 列名应该与原来的Hive表相同，但你可以使用这些列中的一些，并以任何顺序，你也可以使用一些从其他列计算的别名列。 +- 列类型与原Hive表的列类型保持一致。 +- “Partition by expression”应与原Hive表保持一致，“Partition by expression”中的列应在表结构中。 + +**引擎参数** + +- `thrift://host:port` — Hive Metastore 地址 + +- `database` — 远程数据库名. + +- `table` — 远程数据表名. + +## 使用示例 {#usage-example} + +### 如何使用HDFS文件系统的本地缓存 +我们强烈建议您为远程文件系统启用本地缓存。基准测试显示，如果使用缓存，它的速度会快两倍。 + +在使用缓存之前，请将其添加到 `config.xml` +``` xml + + true + local_cache + 559096952 + 1048576 + +``` + + +- enable: 开启后，ClickHouse将为HDFS (远程文件系统)维护本地缓存。 +- root_dir: 必需的。用于存储远程文件系统的本地缓存文件的根目录。 +- limit_size: 必需的。本地缓存文件的最大大小(单位为字节)。 +- bytes_read_before_flush: 从远程文件系统下载文件时，刷新到本地文件系统前的控制字节数。缺省值为1MB。 + +当ClickHouse为远程文件系统启用了本地缓存时，用户仍然可以选择不使用缓存，并在查询中设置`use_local_cache_for_remote_fs = 0 `, `use_local_cache_for_remote_fs` 默认为 `false`。 + +### 查询 ORC 输入格式的Hive 表 + +#### 在 Hive 中建表 +``` text +hive > CREATE TABLE `test`.`test_orc`( + `f_tinyint` tinyint, + `f_smallint` smallint, + `f_int` int, + `f_integer` int, + `f_bigint` bigint, + `f_float` float, + `f_double` double, + `f_decimal` decimal(10,0), + `f_timestamp` timestamp, + `f_date` date, + `f_string` string, + `f_varchar` varchar(100), + `f_bool` boolean, + `f_binary` binary, + `f_array_int` array, + `f_array_string` array, + `f_array_float` array, + `f_array_array_int` array>, + `f_array_array_string` array>, + `f_array_array_float` array>) +PARTITIONED BY ( + `day` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.orc.OrcSerde' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat' +LOCATION + 'hdfs://testcluster/data/hive/test.db/test_orc' + +OK +Time taken: 0.51 seconds + +hive > insert into test.test_orc partition(day='2021-09-18') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, current_timestamp(), current_date(), 'hello world', 'hello world', 'hello world', true, 'hello world', array(1, 2, 3), array('hello world', 'hello world'), array(float(1.1), float(1.2)), array(array(1, 2), array(3, 4)), array(array('a', 'b'), array('c', 'd')), array(array(float(1.11), float(2.22)), array(float(3.33), float(4.44))); +OK +Time taken: 36.025 seconds + +hive > select * from test.test_orc; +OK +1 2 3 4 5 6.11 7.22 8 2021-11-05 12:38:16.314 2021-11-05 hello world hello world hello world true hello world [1,2,3] ["hello world","hello world"] [1.1,1.2] [[1,2],[3,4]] [["a","b"],["c","d"]] [[1.11,2.22],[3.33,4.44]] 2021-09-18 +Time taken: 0.295 seconds, Fetched: 1 row(s) +``` + +#### 在 ClickHouse 中建表 + +ClickHouse中的表，从上面创建的Hive表中获取数据: + +``` sql +CREATE TABLE test.test_orc +( + `f_tinyint` Int8, + `f_smallint` Int16, + `f_int` Int32, + `f_integer` Int32, + `f_bigint` Int64, + `f_float` Float32, + `f_double` Float64, + `f_decimal` Float64, + `f_timestamp` DateTime, + `f_date` Date, + `f_string` String, + `f_varchar` String, + `f_bool` Bool, + `f_binary` String, + `f_array_int` Array(Int32), + `f_array_string` Array(String), + `f_array_float` Array(Float32), + `f_array_array_int` Array(Array(Int32)), + `f_array_array_string` Array(Array(String)), + `f_array_array_float` Array(Array(Float32)), + `day` String +) +ENGINE = Hive('thrift://202.168.117.26:9083', 'test', 'test_orc') +PARTITION BY day + +``` + +``` sql +SELECT * FROM test.test_orc settings input_format_orc_allow_missing_columns = 1\G +``` + +``` text +SELECT * +FROM test.test_orc +SETTINGS input_format_orc_allow_missing_columns = 1 + +Query id: c3eaffdc-78ab-43cd-96a4-4acc5b480658 + +Row 1: +────── +f_tinyint: 1 +f_smallint: 2 +f_int: 3 +f_integer: 4 +f_bigint: 5 +f_float: 6.11 +f_double: 7.22 +f_decimal: 8 +f_timestamp: 2021-12-04 04:00:44 +f_date: 2021-12-03 +f_string: hello world +f_varchar: hello world +f_bool: true +f_binary: hello world +f_array_int: [1,2,3] +f_array_string: ['hello world','hello world'] +f_array_float: [1.1,1.2] +f_array_array_int: [[1,2],[3,4]] +f_array_array_string: [['a','b'],['c','d']] +f_array_array_float: [[1.11,2.22],[3.33,4.44]] +day: 2021-09-18 + + +1 rows in set. Elapsed: 0.078 sec. +``` + +### 查询 Parquest 输入格式的Hive 表 + +#### 在 Hive 中建表 +``` text +hive > +CREATE TABLE `test`.`test_parquet`( + `f_tinyint` tinyint, + `f_smallint` smallint, + `f_int` int, + `f_integer` int, + `f_bigint` bigint, + `f_float` float, + `f_double` double, + `f_decimal` decimal(10,0), + `f_timestamp` timestamp, + `f_date` date, + `f_string` string, + `f_varchar` varchar(100), + `f_char` char(100), + `f_bool` boolean, + `f_binary` binary, + `f_array_int` array, + `f_array_string` array, + `f_array_float` array, + `f_array_array_int` array>, + `f_array_array_string` array>, + `f_array_array_float` array>) +PARTITIONED BY ( + `day` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat' +LOCATION + 'hdfs://testcluster/data/hive/test.db/test_parquet' +OK +Time taken: 0.51 seconds + +hive > insert into test.test_parquet partition(day='2021-09-18') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, current_timestamp(), current_date(), 'hello world', 'hello world', 'hello world', true, 'hello world', array(1, 2, 3), array('hello world', 'hello world'), array(float(1.1), float(1.2)), array(array(1, 2), array(3, 4)), array(array('a', 'b'), array('c', 'd')), array(array(float(1.11), float(2.22)), array(float(3.33), float(4.44))); +OK +Time taken: 36.025 seconds + +hive > select * from test.test_parquet; +OK +1 2 3 4 5 6.11 7.22 8 2021-12-14 17:54:56.743 2021-12-14 hello world hello world hello world true hello world [1,2,3] ["hello world","hello world"] [1.1,1.2] [[1,2],[3,4]] [["a","b"],["c","d"]] [[1.11,2.22],[3.33,4.44]] 2021-09-18 +Time taken: 0.766 seconds, Fetched: 1 row(s) +``` + +#### 在 ClickHouse 中建表 + +ClickHouse 中的表， 从上面创建的Hive表中获取数据: + +``` sql +CREATE TABLE test.test_parquet +( + `f_tinyint` Int8, + `f_smallint` Int16, + `f_int` Int32, + `f_integer` Int32, + `f_bigint` Int64, + `f_float` Float32, + `f_double` Float64, + `f_decimal` Float64, + `f_timestamp` DateTime, + `f_date` Date, + `f_string` String, + `f_varchar` String, + `f_char` String, + `f_bool` Bool, + `f_binary` String, + `f_array_int` Array(Int32), + `f_array_string` Array(String), + `f_array_float` Array(Float32), + `f_array_array_int` Array(Array(Int32)), + `f_array_array_string` Array(Array(String)), + `f_array_array_float` Array(Array(Float32)), + `day` String +) +ENGINE = Hive('thrift://localhost:9083', 'test', 'test_parquet') +PARTITION BY day +``` + +``` sql +SELECT * FROM test.test_parquet settings input_format_parquet_allow_missing_columns = 1\G +``` + +``` text +SELECT * +FROM test_parquet +SETTINGS input_format_parquet_allow_missing_columns = 1 + +Query id: 4e35cf02-c7b2-430d-9b81-16f438e5fca9 + +Row 1: +────── +f_tinyint: 1 +f_smallint: 2 +f_int: 3 +f_integer: 4 +f_bigint: 5 +f_float: 6.11 +f_double: 7.22 +f_decimal: 8 +f_timestamp: 2021-12-14 17:54:56 +f_date: 2021-12-14 +f_string: hello world +f_varchar: hello world +f_char: hello world +f_bool: true +f_binary: hello world +f_array_int: [1,2,3] +f_array_string: ['hello world','hello world'] +f_array_float: [1.1,1.2] +f_array_array_int: [[1,2],[3,4]] +f_array_array_string: [['a','b'],['c','d']] +f_array_array_float: [[1.11,2.22],[3.33,4.44]] +day: 2021-09-18 + +1 rows in set. Elapsed: 0.357 sec. +``` + +### 查询文本输入格式的Hive表 + +#### 在Hive 中建表 + +``` text +hive > +CREATE TABLE `test`.`test_text`( + `f_tinyint` tinyint, + `f_smallint` smallint, + `f_int` int, + `f_integer` int, + `f_bigint` bigint, + `f_float` float, + `f_double` double, + `f_decimal` decimal(10,0), + `f_timestamp` timestamp, + `f_date` date, + `f_string` string, + `f_varchar` varchar(100), + `f_char` char(100), + `f_bool` boolean, + `f_binary` binary, + `f_array_int` array, + `f_array_string` array, + `f_array_float` array, + `f_array_array_int` array>, + `f_array_array_string` array>, + `f_array_array_float` array>) +PARTITIONED BY ( + `day` string) +ROW FORMAT SERDE + 'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe' +STORED AS INPUTFORMAT + 'org.apache.hadoop.mapred.TextInputFormat' +OUTPUTFORMAT + 'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat' +LOCATION + 'hdfs://testcluster/data/hive/test.db/test_text' +Time taken: 0.1 seconds, Fetched: 34 row(s) + + +hive > insert into test.test_text partition(day='2021-09-18') select 1, 2, 3, 4, 5, 6.11, 7.22, 8.333, current_timestamp(), current_date(), 'hello world', 'hello world', 'hello world', true, 'hello world', array(1, 2, 3), array('hello world', 'hello world'), array(float(1.1), float(1.2)), array(array(1, 2), array(3, 4)), array(array('a', 'b'), array('c', 'd')), array(array(float(1.11), float(2.22)), array(float(3.33), float(4.44))); +OK +Time taken: 36.025 seconds + +hive > select * from test.test_text; +OK +1 2 3 4 5 6.11 7.22 8 2021-12-14 18:11:17.239 2021-12-14 hello world hello world hello world true hello world [1,2,3] ["hello world","hello world"] [1.1,1.2] [[1,2],[3,4]] [["a","b"],["c","d"]] [[1.11,2.22],[3.33,4.44]] 2021-09-18 +Time taken: 0.624 seconds, Fetched: 1 row(s) +``` + +#### 在 ClickHouse 中建表 + + +ClickHouse中的表， 从上面创建的Hive表中获取数据: +``` sql +CREATE TABLE test.test_text +( + `f_tinyint` Int8, + `f_smallint` Int16, + `f_int` Int32, + `f_integer` Int32, + `f_bigint` Int64, + `f_float` Float32, + `f_double` Float64, + `f_decimal` Float64, + `f_timestamp` DateTime, + `f_date` Date, + `f_string` String, + `f_varchar` String, + `f_char` String, + `f_bool` Bool, + `day` String +) +ENGINE = Hive('thrift://localhost:9083', 'test', 'test_text') +PARTITION BY day +``` + +``` sql +SELECT * FROM test.test_text settings input_format_skip_unknown_fields = 1, input_format_with_names_use_header = 1, date_time_input_format = 'best_effort'\G +``` + +``` text +SELECT * +FROM test.test_text +SETTINGS input_format_skip_unknown_fields = 1, input_format_with_names_use_header = 1, date_time_input_format = 'best_effort' + +Query id: 55b79d35-56de-45b9-8be6-57282fbf1f44 + +Row 1: +────── +f_tinyint: 1 +f_smallint: 2 +f_int: 3 +f_integer: 4 +f_bigint: 5 +f_float: 6.11 +f_double: 7.22 +f_decimal: 8 +f_timestamp: 2021-12-14 18:11:17 +f_date: 2021-12-14 +f_string: hello world +f_varchar: hello world +f_char: hello world +f_bool: true +day: 2021-09-18 +``` diff --git a/docs/zh/engines/table-engines/integrations/index.md b/docs/zh/engines/table-engines/integrations/index.md index 0c34ae078a0..5ed4a555f9c 100644 --- a/docs/zh/engines/table-engines/integrations/index.md +++ b/docs/zh/engines/table-engines/integrations/index.md @@ -19,3 +19,5 @@ ClickHouse 提供了多种方式来与外部系统集成，包括表引擎。像 - [EmbeddedRocksDB](../../../engines/table-engines/integrations/embedded-rocksdb.md) - [RabbitMQ](../../../engines/table-engines/integrations/rabbitmq.md) - [PostgreSQL](../../../engines/table-engines/integrations/postgresql.md) +- [SQLite](../../../engines/table-engines/integrations/sqlite.md) +- [Hive](../../../engines/table-engines/integrations/hive.md) diff --git a/docs/zh/faq/general/dbms-naming.md b/docs/zh/faq/general/dbms-naming.md deleted file mode 120000 index 0df856af0ca..00000000000 --- a/docs/zh/faq/general/dbms-naming.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/general/dbms-naming.md \ No newline at end of file diff --git a/docs/zh/faq/general/dbms-naming.md b/docs/zh/faq/general/dbms-naming.md new file mode 100644 index 00000000000..8d4353f9322 --- /dev/null +++ b/docs/zh/faq/general/dbms-naming.md @@ -0,0 +1,17 @@ +--- +title: "\u201CClickHouse\u201D 有什么含义?" +toc_hidden: true +toc_priority: 10 +--- + +# “ClickHouse” 有什么含义? {#what-does-clickhouse-mean} + +它是“**点击**流”和“数据**仓库**”的组合。它来自于Yandex最初的用例。在Metrica网站上，ClickHouse本应该保存人们在互联网上的所有点击记录，现在它仍然在做这项工作。你可以在[ClickHouse history](../../introduction/history.md)页面上阅读更多关于这个用例的信息。 + +这个由两部分组成的意思有两个结果: + +- 唯一正确的写“Click**H** house”的方式是用大写H。 +- 如果需要缩写，请使用“**CH**”。由于一些历史原因，缩写CK在中国也很流行，主要是因为中文中最早的一个关于ClickHouse的演讲使用了这种形式。 + +!!! info “有趣的事实” + 多年后ClickHouse闻名于世, 这种命名方法：结合各有深意的两个词被赞扬为最好的数据库命名方式, 卡内基梅隆大学数据库副教授[Andy Pavlo做的研究](https://www.cs.cmu.edu/~pavlo/blog/2020/03/on-naming-a-database-management-system.html) 。ClickHouse与Postgres共同获得“史上最佳数据库名”奖。 diff --git a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md deleted file mode 120000 index 5ac9a615386..00000000000 --- a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/general/how-do-i-contribute-code-to-clickhouse.md \ No newline at end of file diff --git a/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md new file mode 100644 index 00000000000..39d2d639229 --- /dev/null +++ b/docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md @@ -0,0 +1,17 @@ +--- +title: 我如何为ClickHouse贡献代码? +toc_hidden: true +toc_priority: 120 +--- + +# 我如何为ClickHouse贡献代码? {#how-do-i-contribute-code-to-clickhouse} + +ClickHouse是一个开源项目[在GitHub上开发](https://github.com/ClickHouse/ClickHouse)。 + +按照惯例，贡献指南发布在源代码库根目录的 [CONTRIBUTING.md](https://github.com/ClickHouse/ClickHouse/blob/master/CONTRIBUTING.md)文件中。 + +如果你想对ClickHouse提出实质性的改变建议，可以考虑[在GitHub上发布一个问题](https://github.com/ClickHouse/ClickHouse/issues/new/choose)，解释一下你想做什么，先与维护人员和社区讨论一下。[此类RFC问题的例子](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aissue+is%3Aopen+rfc)。 + +如果您的贡献与安全相关，也请查看[我们的安全政策](https://github.com/ClickHouse/ClickHouse/security/policy/)。 + + diff --git a/docs/zh/faq/index.md b/docs/zh/faq/index.md index 9887d2c6c0a..1ba1b792fbd 100644 --- a/docs/zh/faq/index.md +++ b/docs/zh/faq/index.md @@ -26,6 +26,7 @@ toc_priority: 76 - **[运维操作](../faq/operations/index.md)** - [如果想在生产环境部署，需要用哪个版本的 ClickHouse 呢?](../faq/operations/production.md) - [是否可能从 ClickHouse 数据表中删除所有旧的数据记录?](../faq/operations/delete-old-data.md) + - [ClickHouse支持多区域复制吗?](../faq/operations/multi-region-replication.md) - **[集成开发](../faq/integration/index.md)** - [如何从 ClickHouse 导出数据到一个文件?](../faq/integration/file-export.md) - [如果我用ODBC链接Oracle数据库出现编码问题该怎么办?](../faq/integration/oracle-odbc.md) diff --git a/docs/zh/faq/integration/index.md b/docs/zh/faq/integration/index.md deleted file mode 120000 index 8323d6218a3..00000000000 --- a/docs/zh/faq/integration/index.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/integration/index.md \ No newline at end of file diff --git a/docs/zh/faq/integration/index.md b/docs/zh/faq/integration/index.md new file mode 100644 index 00000000000..2bfd728ec8c --- /dev/null +++ b/docs/zh/faq/integration/index.md @@ -0,0 +1,21 @@ +--- +title: 关于集成ClickHouse和其他系统的问题 +toc_hidden_folder: true +toc_priority: 4 +toc_title: Integration +--- + +# 关于集成ClickHouse和其他系统的问题 {#question-about-integrating-clickhouse-and-other-systems} + +问题: + +- [如何从 ClickHouse 导出数据到一个文件?](../../faq/integration/file-export.md) +- [如何导入JSON到ClickHouse?](../../faq/integration/json-import.md) +- [如果我用ODBC链接Oracle数据库出现编码问题该怎么办?](../../faq/integration/oracle-odbc.md) + + + +!!! info "没看到你要找的东西吗?" + 查看[其他faq类别](../../faq/index.md)或浏览左边栏中的主要文档文章。 + +{## [原文](https://clickhouse.com/docs/en/faq/integration/) ##} \ No newline at end of file diff --git a/docs/zh/faq/operations/index.md b/docs/zh/faq/operations/index.md deleted file mode 120000 index fd141164fdc..00000000000 --- a/docs/zh/faq/operations/index.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/operations/index.md \ No newline at end of file diff --git a/docs/zh/faq/operations/index.md b/docs/zh/faq/operations/index.md new file mode 100644 index 00000000000..cdf4b9622ec --- /dev/null +++ b/docs/zh/faq/operations/index.md @@ -0,0 +1,20 @@ +--- +title: 关于操作ClickHouse服务器和集群的问题 +toc_hidden_folder: true +toc_priority: 3 +toc_title: Operations +--- + +# 关于操作ClickHouse服务器和集群的问题 {#question-about-operating-clickhouse-servers-and-clusters} + +问题: + +- [如果想在生产环境部署，需要用哪个版本的 ClickHouse 呢?](../../faq/operations/production.md) +- [是否可能从 ClickHouse 数据表中删除所有旧的数据记录?](../../faq/operations/delete-old-data.md) +- [ClickHouse支持多区域复制吗?](../../faq/operations/multi-region-replication.md) + + +!!! info "没看到你要找的东西吗?" + 查看[其他faq类别](../../faq/index.md)或浏览左边栏中的主要文档文章。 + +{## [原文](https://clickhouse.com/docs/en/faq/production/) ##} diff --git a/docs/zh/faq/operations/multi-region-replication.md b/docs/zh/faq/operations/multi-region-replication.md deleted file mode 120000 index dbc985ee1fb..00000000000 --- a/docs/zh/faq/operations/multi-region-replication.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/faq/operations/multi-region-replication.md \ No newline at end of file diff --git a/docs/zh/faq/operations/multi-region-replication.md b/docs/zh/faq/operations/multi-region-replication.md new file mode 100644 index 00000000000..f5ab147bde6 --- /dev/null +++ b/docs/zh/faq/operations/multi-region-replication.md @@ -0,0 +1,14 @@ +--- +title: ClickHouse支持多区域复制吗? +toc_hidden: true +toc_priority: 30 +--- + +# ClickHouse支持多区域复制吗? {#does-clickhouse-support-multi-region-replication} + +简短的回答是“是的”。然而，我们建议将所有区域/数据中心之间的延迟保持在两位数字范围内，否则，在通过分布式共识协议时，写性能将受到影响。例如，美国海岸之间的复制可能会很好，但美国和欧洲之间就不行。 + +在配置方面，这与单区域复制没有区别，只是使用位于不同位置的主机作为副本。 + +更多信息，请参见[关于数据复制的完整文章](../../engines/table-engines/mergetree-family/replication.md)。 + diff --git a/docs/zh/interfaces/http.md b/docs/zh/interfaces/http.md index 738b0365f46..16f51eac9a8 100644 --- a/docs/zh/interfaces/http.md +++ b/docs/zh/interfaces/http.md @@ -18,6 +18,17 @@ $ curl 'http://localhost:8123/' Ok. ``` +Web UI 可以通过这个地址访问: `http://localhost:8123/play`. +在运行状况检查脚本中，使用`GET /ping`请求。这个处理方法总是返回 "Ok"。(以换行结尾)。可从18.12.13版获得。请参见' /replicas_status '检查复制集的延迟。 + + +``` bash +$ curl 'http://localhost:8123/ping' +Ok. +$ curl 'http://localhost:8123/replicas_status' +Ok. +``` + 通过URL中的 `query` 参数来发送请求，或者发送POST请求，或者将查询的开头部分放在URL的`query`参数中，其他部分放在POST中（我们会在后面解释为什么这样做是有必要的）。URL的大小会限制在16KB，所以发送大型查询时要时刻记住这点。 如果请求成功，将会收到200的响应状态码和响应主体中的结果。 diff --git a/docs/zh/operations/requirements.md b/docs/zh/operations/requirements.md index c3013f738a2..964d7aa34f4 100644 --- a/docs/zh/operations/requirements.md +++ b/docs/zh/operations/requirements.md @@ -1,59 +1,59 @@ --- toc_priority: 44 -toc_title: "要求" +toc_title: "必备条件" --- -# 要求 {#requirements} +# 必备条件 {#requirements} ## CPU {#cpu} -对于从预构建的deb包进行安装，请使用具有x86_64架构并支持SSE4.2指令的CPU。 要使用不支持SSE4.2或具有AArch64或PowerPC64LE体系结构的处理器运行ClickHouse，您应该从源代码构建ClickHouse。 +如果您使用预编译的DEB/RPM包安装ClickHouse，请使用支持SSE4.2指令集的x86_64架构的CPU。如果需要在不支持SSE4.2指令集的CPU上，或者在AArch64（ARM）和PowerPC64LE（IBM Power）架构上运行ClickHouse，您应该从源码编译ClickHouse。 -ClickHouse实现并行数据处理并使用所有可用的硬件资源。 在选择处理器时，考虑到ClickHouse在具有大量内核但时钟速率较低的配置中的工作效率要高于具有较少内核和较高时钟速率的配置。 例如，具有2600MHz的16核心优于具有3600MHz的8核心。 +ClickHouse实现了并行数据处理，处理时会使用所有的可用资源。在选择处理器时，请注意：ClickHouse在具有大量计算核、时钟频率稍低的平台上比计算核少、时钟频率高的平台上效率更高。例如，ClickHouse在16核 2.6GHz的CPU上运行速度高于8核 3.6GHz的CPU。 -建议使用 **睿频加速** 和 **超线程** 技术。 它显着提高了典型工作负载的性能。 +建议使用 **睿频加速** 和 **超线程** 技术。 它显着提高了正常工作负载的性能。 ## RAM {#ram} -我们建议使用至少4GB的RAM来执行重要的查询。 ClickHouse服务器可以使用少得多的RAM运行，但它需要处理查询的内存。 +我们建议使用至少4GB的内存来执行重要的查询。 ClickHouse服务器可以使用很少的内存运行，但它需要一定量的内存用于处理查询。 -RAM所需的体积取决于: +ClickHouse所需内存取决于: -- 查询的复杂性。 -- 查询中处理的数据量。 +- 查询的复杂程度。 +- 查询处理的数据量。 -要计算所需的RAM体积，您应该估计临时数据的大小 [GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause), [DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct), [JOIN](../sql-reference/statements/select/join.md#select-join) 和您使用的其他操作。 +要计算所需的内存大小，您应该考虑用于[GROUP BY](../sql-reference/statements/select/group-by.md#select-group-by-clause)、[DISTINCT](../sql-reference/statements/select/distinct.md#select-distinct)、[JOIN](../sql-reference/statements/select/join.md#select-join) 和其他操作所需的临时数据量。 -ClickHouse可以使用外部存储器来存储临时数据。看 [在外部存储器中分组](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory) 有关详细信息。 +ClickHouse可以使用外部存储器来存储临时数据。详情请见[在外部存储器中分组](../sql-reference/statements/select/group-by.md#select-group-by-in-external-memory)。 ## 交换文件 {#swap-file} -禁用生产环境的交换文件。 +请在生产环境禁用交换文件。 ## 存储子系统 {#storage-subsystem} 您需要有2GB的可用磁盘空间来安装ClickHouse。 -数据所需的存储量应单独计算。 评估应包括: +数据所需的存储空间应单独计算。预估存储容量时请考虑: -- 估计数据量。 +- 数据量 - 您可以采取数据的样本并从中获取行的平均大小。 然后将该值乘以计划存储的行数。 + 您可以对数据进行采样并计算每行的平均占用空间。然后将该值乘以计划存储的行数。 -- 数据压缩系数。 +- 数据压缩比 - 要估计数据压缩系数，请将数据的样本加载到ClickHouse中，并将数据的实际大小与存储的表的大小进行比较。 例如，点击流数据通常被压缩6-10倍。 + 要计算数据压缩比，请将样本数据写入ClickHouse，并将原始数据大小与ClickHouse实际存储的数据进行比较。例如，用户点击行为的原始数据压缩比通常为6-10。 -要计算要存储的最终数据量，请将压缩系数应用于估计的数据量。 如果计划将数据存储在多个副本中，则将估计的量乘以副本数。 +请将原始数据的大小除以压缩比来获得实际所需存储的大小。如果您打算将数据存放于几个副本中，请将存储容量乘上副本数。 ## 网络 {#network} -如果可能的话，使用10G或更高级别的网络。 +如果可能的话，请使用10G或更高级别的网络。 -网络带宽对于处理具有大量中间结果数据的分布式查询至关重要。 此外，网络速度会影响复制过程。 +网络带宽对于处理具有大量中间结果数据的分布式查询至关重要。此外，网络速度会影响复制过程。 ## 软件 {#software} -ClickHouse主要是为Linux系列操作系统开发的。 推荐的Linux发行版是Ubuntu。 `tzdata` 软件包应安装在系统中。 +ClickHouse主要是为Linux系列操作系统开发的。推荐的Linux发行版是Ubuntu。您需要检查`tzdata`（对于Ubuntu）软件包是否在安装ClickHouse之前已经安装。 -ClickHouse也可以在其他操作系统系列中工作。 查看详细信息 [开始](../getting-started/index.md) 文档的部分。 +ClickHouse也可以在其他操作系统系列中工作。详情请查看[开始](../getting-started/index.md)。 diff --git a/docs/zh/operations/system-tables/query_thread_log.md b/docs/zh/operations/system-tables/query_thread_log.md index 33583f3b730..64f9ed27393 100644 --- a/docs/zh/operations/system-tables/query_thread_log.md +++ b/docs/zh/operations/system-tables/query_thread_log.md @@ -1,67 +1,62 @@ ---- -machine_translated: true -machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3 ---- - # 系统。query_thread_log {#system_tables-query_thread_log} 包含有关执行查询的线程的信息，例如，线程名称、线程开始时间、查询处理的持续时间。 -开始记录: +开启日志功能: -1. 在配置参数 [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) 科。 -2. 设置 [log_query_threads](../../operations/settings/settings.md#settings-log-query-threads) 到1。 +1. 在配置参数 [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) 部分。 +2. 设置 [log_query_threads](../../operations/settings/settings.md#settings-log-query-threads) 为1。 -数据的冲洗周期设置在 `flush_interval_milliseconds` 的参数 [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) 服务器设置部分。 要强制冲洗，请使用 [SYSTEM FLUSH LOGS](../../sql-reference/statements/system.md#query_language-system-flush_logs) 查询。 +数据从缓存写入数据表周期时间参数 `flush_interval_milliseconds` 位于 [query_thread_log](../../operations/server-configuration-parameters/settings.md#server_configuration_parameters-query_thread_log) 服务器设置部分。如果需要强制从缓存写入数据表，请使用 [SYSTEM FLUSH LOGS](../../sql-reference/statements/system.md#query_language-system-flush_logs) 查询请求。 -ClickHouse不会自动从表中删除数据。 看 [导言](../../operations/system-tables/index.md#system-tables-introduction) 欲了解更多详情。 +ClickHouse不会自动从表中删除数据。 欲了解更多详情，请参照 [介绍](../../operations/system-tables/index.md#system-tables-introduction)。 列: -- `event_date` ([日期](../../sql-reference/data-types/date.md)) — The date when the thread has finished execution of the query. -- `event_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — The date and time when the thread has finished execution of the query. -- `query_start_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — Start time of query execution. -- `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Duration of query execution. -- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read rows. -- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Number of read bytes. -- `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` 查询，写入的行数。 对于其他查询，列值为0。 -- `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — For `INSERT` 查询时，写入的字节数。 对于其他查询，列值为0。 -- `memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — The difference between the amount of allocated and freed memory in context of this thread. -- `peak_memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — The maximum difference between the amount of allocated and freed memory in context of this thread. -- `thread_name` ([字符串](../../sql-reference/data-types/string.md)) — Name of the thread. -- `thread_number` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Internal thread ID. -- `thread_id` ([Int32](../../sql-reference/data-types/int-uint.md)) — thread ID. -- `master_thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — OS initial ID of initial thread. -- `query` ([字符串](../../sql-reference/data-types/string.md)) — Query string. -- `is_initial_query` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Query type. Possible values: - - 1 — Query was initiated by the client. - - 0 — Query was initiated by another query for distributed query execution. -- `user` ([字符串](../../sql-reference/data-types/string.md)) — Name of the user who initiated the current query. -- `query_id` ([字符串](../../sql-reference/data-types/string.md)) — ID of the query. -- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that was used to make the query. -- `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the query. -- `initial_user` ([字符串](../../sql-reference/data-types/string.md)) — Name of the user who ran the initial query (for distributed query execution). -- `initial_query_id` ([字符串](../../sql-reference/data-types/string.md)) — ID of the initial query (for distributed query execution). -- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — IP address that the parent query was launched from. -- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — The client port that was used to make the parent query. -- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — Interface that the query was initiated from. Possible values: +- `event_date` ([日期](../../sql-reference/data-types/date.md)) — 该查询线程执行完成的日期。 +- `event_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — 该查询线程执行完成的时间。 +- `query_start_time` ([日期时间](../../sql-reference/data-types/datetime.md)) — 查询的开始时间。 +- `query_duration_ms` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 查询执行持续的时间。 +- `read_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 读取的行数。 +- `read_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 读取的字节数。 +- `written_rows` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 对于 `INSERT` 查询，写入的行数。 对于其他查询，为0。 +- `written_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 对于 `INSERT` 查询，写入的字节数。 对于其他查询，为0。 +- `memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — 在线程上下文，分配的内存和空闲内存之差。 +- `peak_memory_usage` ([Int64](../../sql-reference/data-types/int-uint.md)) — 在线程上下文，分配的内存和空闲内存之差的最大值。 +- `thread_name` ([字符串](../../sql-reference/data-types/string.md)) — 线程名。 +- `thread_number` ([UInt32](../../sql-reference/data-types/int-uint.md)) — 内部线程ID。 +- `thread_id` ([Int32](../../sql-reference/data-types/int-uint.md)) — 线程ID。 +- `master_thread_id` ([UInt64](../../sql-reference/data-types/int-uint.md#uint-ranges)) — OS初始线程的初始ID。 +- `query` ([字符串](../../sql-reference/data-types/string.md)) — 查询语句。 +- `is_initial_query` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 查询类型，可能的值： + - 1 — 由用户发起的查询。 + - 0 — 由其他查询发起的分布式查询。 +- `user` ([字符串](../../sql-reference/data-types/string.md)) — 发起查询的用户名。 +- `query_id` ([字符串](../../sql-reference/data-types/string.md)) — 查询的ID。 +- `address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起查询的IP地址。 +- `port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的端口。 +- `initial_user` ([字符串](../../sql-reference/data-types/string.md)) — 首次发起查询的用户名（对于分布式查询）。 +- `initial_query_id` ([字符串](../../sql-reference/data-types/string.md)) — 首次发起查询的ID（对于分布式查询）。 +- `initial_address` ([IPv6](../../sql-reference/data-types/domains/ipv6.md)) — 发起该查询的父查询IP地址。 +- `initial_port` ([UInt16](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起该查询的父查询端口。 +- `interface` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的界面，可能的值: - 1 — TCP. - 2 — HTTP. -- `os_user` ([字符串](../../sql-reference/data-types/string.md)) — OS's username who runs [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md). -- `client_hostname` ([字符串](../../sql-reference/data-types/string.md)) — Hostname of the client machine where the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或者运行另一个TCP客户端。 -- `client_name` ([字符串](../../sql-reference/data-types/string.md)) — The [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端名称。 -- `client_revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Revision of the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端。 -- `client_version_major` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Major version of the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端。 -- `client_version_minor` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Minor version of the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端。 -- `client_version_patch` ([UInt32](../../sql-reference/data-types/int-uint.md)) — Patch component of the [ﾂ环板clientｮﾂ嘉ｯﾂ偲](../../interfaces/cli.md) 或另一个TCP客户端版本。 -- `http_method` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — HTTP method that initiated the query. Possible values: - - 0 — The query was launched from the TCP interface. +- `os_user` ([字符串](../../sql-reference/data-types/string.md)) — 使用 [clickhouse-client](../../interfaces/cli.md) 的系统用户名。 +- `client_hostname` ([字符串](../../sql-reference/data-types/string.md)) — 运行 [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的主机名。 +- `client_name` ([字符串](../../sql-reference/data-types/string.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的名称。 +- `client_revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的修订号。 +- `client_version_major` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的主版本号。 +- `client_version_minor` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的次版本号。 +- `client_version_patch` ([UInt32](../../sql-reference/data-types/int-uint.md)) — [clickhouse-client](../../interfaces/cli.md) 或另一个TCP客户端的补丁版本号。 +- `http_method` ([UInt8](../../sql-reference/data-types/int-uint.md#uint-ranges)) — 发起查询的HTTP方法，可能的值： + - 0 — 查询通过TCP界面发起。 - 1 — `GET` 方法被使用。 - 2 — `POST` 方法被使用。 -- `http_user_agent` ([字符串](../../sql-reference/data-types/string.md)) — The `UserAgent` http请求中传递的标头。 -- `quota_key` ([字符串](../../sql-reference/data-types/string.md)) — The “quota key” 在指定 [配额](../../operations/quotas.md) 设置（见 `keyed`). -- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse revision. -- `ProfileEvents` ([数组（字符串, UInt64)](../../sql-reference/data-types/array.md)) — Counters that measure different metrics for this thread. The description of them could be found in the table [系统。活动](#system_tables-events). +- `http_user_agent` ([字符串](../../sql-reference/data-types/string.md)) — `UserAgent` HTTP请求中传递的UA表头。 +- `quota_key` ([字符串](../../sql-reference/data-types/string.md)) — “quota key” 在 [配额](../../operations/quotas.md) 设置内（详见 `keyed`). +- `revision` ([UInt32](../../sql-reference/data-types/int-uint.md)) — ClickHouse 修订版本号. +- `ProfileEvents` ([数组（字符串, UInt64)](../../sql-reference/data-types/array.md)) — 对于该线程的多个指标计数器。这一项可以参考 [system.events](#system_tables-events). **示例** @@ -113,4 +108,5 @@ ProfileEvents: {'Query':1,'SelectQuery':1,'ReadCompressedBytes':36,'Compr **另请参阅** -- [系统。query_log](../../operations/system-tables/query_log.md#system_tables-query_log) — Description of the `query_log` 系统表，其中包含有关查询执行的公共信息。 +- [system.query_log](../../operations/system-tables/query_log.md#system_tables-query_log) — `query_log` 系统表描述，其中包含有关查询执行的公共信息。 +- [system.query_views_log](../../operations/system-tables/query_views_log.md#system_tables-query_views_log) — 这个表包含在查询线程中使用的各个视图的信息。 diff --git a/docs/zh/sql-reference/ansi.md b/docs/zh/sql-reference/ansi.md index 0e7fa1d06c3..5aad2cf52a8 100644 --- a/docs/zh/sql-reference/ansi.md +++ b/docs/zh/sql-reference/ansi.md @@ -1,180 +1,189 @@ --- -machine_translated: true -machine_translated_rev: ad252bbb4f7e2899c448eb42ecc39ff195c8faa1 toc_priority: 40 toc_title: "ANSI\u517C\u5BB9\u6027" --- -# Ansi Sql兼容性的ClickHouse SQL方言 {#ansi-sql-compatibility-of-clickhouse-sql-dialect} +# ClickHouse SQL方言 与ANSI SQL的兼容性{#ansi-sql-compatibility-of-clickhouse-sql-dialect} !!! note "注" - 本文依赖于表38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013. + 本文参考Annex G所著的[ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8)标准. ## 行为差异 {#differences-in-behaviour} -下表列出了查询功能在ClickHouse中有效但不符合ANSI SQL标准的情况。 +下表列出了ClickHouse能够使用，但与ANSI SQL规定有差异的查询特性。 -| Feature ID | 功能名称 | 差异 | -|------------|--------------------|---------------------------------------------------------------------| -| E011 | 数值（Numeric）数据类型 | 带小数点的数值文字被解释为近似值 (`Float64`）而不是精确值 (`Decimal`) | -| E051-05 | SELECT字段可以重命名 | 字段不仅仅在SELECT结果中可被重命名 | -| E141-01 | 非空约束 | 表中每一列默认为`NOT NULL` | -| E011-04 | 算术运算符 | ClickHouse不会检查算法，并根据自定义规则更改结果数据类型，而是会溢出 | +| 功能ID | 功能名称 | 差异 | +| ------- | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| E011 | 数值型数据类型 | 带小数点的数字被视为近似值 (`Float64`）而不是精确值 (`Decimal`) | +| E051-05 | SELECT 的列可以重命名 | 字段重命名的作用范围不限于进行重命名的SELECT子查询（参考[表达式别名](https://clickhouse.com/docs/zh/sql-reference/syntax/#notes-on-usage)） | +| E141-01 | NOT NULL（非空）约束 | ClickHouse表中每一列默认为`NOT NULL` | +| E011-04 | 算术运算符 | ClickHouse在运算时会进行溢出，而不是四舍五入。此外会根据自定义规则修改结果数据类型（参考[溢出检查](https://clickhouse.com/docs/zh/sql-reference/data-types/decimal/#yi-chu-jian-cha)） | -## 功能匹配 {#feature-status} +## 功能状态 {#feature-status} -| Feature ID | 功能名称 | 匹配 | 评论 | -|------------|----------------------------------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| **E011** | **数字数据类型** | **部分**{.text-warning} | | -| E011-01 | 整型和小型数据类型 | 是 {.text-success} | | -| E011-02 | 真实、双精度和浮点数据类型数据类型 | 部分 {.text-warning} | `FLOAT()`, `REAL` 和 `DOUBLE PRECISION` 不支持 | -| E011-03 | 十进制和数值数据类型 | 部分 {.text-warning} | 只有 `DECIMAL(p,s)` 支持，而不是 `NUMERIC` | -| E011-04 | 算术运算符 | 是 {.text-success} | | -| E011-05 | 数字比较 | 是 {.text-success} | | -| E011-06 | 数字数据类型之间的隐式转换 | 否。 {.text-danger} | ANSI SQL允许在数值类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 | -| **E021** | **字符串类型** | **部分**{.text-warning} | | -| E021-01 | 字符数据类型 | 否。 {.text-danger} | | -| E021-02 | 字符变化数据类型 | 否。 {.text-danger} | `String` 行为类似，但括号中没有长度限制 | -| E021-03 | 字符文字 | 部分 {.text-warning} | 不自动连接连续文字和字符集支持 | -| E021-04 | 字符长度函数 | 部分 {.text-warning} | 非也。 `USING` 条款 | -| E021-05 | OCTET_LENGTH函数 | 非也。 {.text-danger} | `LENGTH` 表现类似 | -| E021-06 | SUBSTRING | 部分 {.text-warning} | 不支持 `SIMILAR` 和 `ESCAPE` 条款，否 `SUBSTRING_REGEX` 备选案文 | -| E021-07 | 字符串联 | 部分 {.text-warning} | 非也。 `COLLATE` 条款 | -| E021-08 | 上下功能 | 是 {.text-success} | | -| E021-09 | 修剪功能 | 是 {.text-success} | | -| E021-10 | 固定长度和可变长度字符串类型之间的隐式转换 | 否。 {.text-danger} | ANSI SQL允许在字符串类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 | -| E021-11 | 职位功能 | 部分 {.text-warning} | 不支持 `IN` 和 `USING` 条款，否 `POSITION_REGEX` 备选案文 | -| E021-12 | 字符比较 | 是 {.text-success} | | -| **E031** | **标识符** | **部分**{.text-warning} | | -| E031-01 | 分隔标识符 | 部分 {.text-warning} | Unicode文字支持有限 | -| E031-02 | 小写标识符 | 是 {.text-success} | | -| E031-03 | 尾部下划线 | 是 {.text-success} | | -| **E051** | **基本查询规范** | **部分**{.text-warning} | | -| E051-01 | SELECT DISTINCT | 是 {.text-success} | | -| E051-02 | GROUP BY子句 | 是 {.text-success} | | -| E051-04 | 分组依据可以包含不在列 ``中出现的列 | 是 {.text-success} | | +| E051-05 | SELECT 的列可以重命名 | 是 {.text-success} | | +| E051-06 | HAVING 从句 | 是 {.text-success} | | +| E051-07 | SELECT 选择的列中允许出现\* | 是 {.text-success} | | +| E051-08 | FROM 从句中的关联名称 | 是 {.text-success} | | +| E051-09 | 重命名 FROM 从句中的列 | 否 {.text-danger} | | +| **E061** | **基本谓词和搜索条件** | **部分**{.text-warning} | | +| E061-01 | 比较谓词 | 是 {.text-success} | | +| E061-02 | BETWEEN 谓词 | 部分 {.text-warning} | 不支持 `SYMMETRIC` 和 `ASYMMETRIC` 从句 | +| E061-03 | IN 谓词后可接值列表 | 是 {.text-success} | | +| E061-04 | LIKE 谓词 | 是 {.text-success} | | +| E061-05 | LIKE 谓词后接 ESCAPE 从句 | 否 {.text-danger} | | +| E061-06 | NULL 谓词 | 是 {.text-success} | | +| E061-07 | 量化比较谓词（ALL、SOME、ANY） | 否 {.text-danger} | | +| E061-08 | EXISTS 谓词 | 否 {.text-danger} | | +| E061-09 | 比较谓词中的子查询 | 是 {.text-success} | | +| E061-11 | IN 谓词中的子查询 | 是 {.text-success} | | +| E061-12 | 量化比较谓词（BETWEEN、IN、LIKE）中的子查询 | 否 {.text-danger} | | +| E061-13 | 相关子查询 | 否 {.text-danger} | | +| E061-14 | 搜索条件 | 是 {.text-success} | | +| **E071** | **基本查询表达式** | **部分**{.text-warning} | | +| E071-01 | UNION DISTINCT 表运算符 | 是 {.text-success} | | +| E071-02 | UNION ALL 表运算符 | 是 {.text-success} | | +| E071-03 | EXCEPT DISTINCT 表运算符 | 否 {.text-danger} | | +| E071-05 | 通过表运算符组合的列不必具有完全相同的数据类型 | 是 {.text-success} | | +| E071-06 | 子查询中的表运算符 | 是 {.text-success} | | +| **E081** | **基本权限** | **是**{.text-success} | | +| E081-01 | 表级别的SELECT（查询）权限 | 是 {.text-success} | | +| E081-02 | DELETE（删除）权限 | 是 {.text-success} | | +| E081-03 | 表级别的INSERT（插入）权限 | 是 {.text-success} | | +| E081-04 | 表级别的UPDATE（更新）权限 | 是 {.text-success} | | +| E081-05 | 列级别的UPDATE（更新）权限 | 是 {.text-success} | | +| E081-06 | 表级别的REFERENCES（引用）权限 | 是 {.text-success} | | +| E081-07 | 列级别的REFERENCES（引用）权限 | 是 {.text-success} | | +| E081-08 | WITH GRANT OPTION | 是 {.text-success} | | +| E081-09 | USAGE（使用）权限 | 是 {.text-success} | | +| E081-10 | EXECUTE（执行）权限 | 是 {.text-success} | | +| **E091** | **集合函数** | **是**{.text-success} | | +| E091-01 | AVG | 是 {.text-success} | | +| E091-02 | COUNT | 是 {.text-success} | | +| E091-03 | MAX | 是 {.text-success} | | +| E091-04 | MIN | 是 {.text-success} | | +| E091-05 | SUM | 是 {.text-success} | | +| E091-06 | ALL修饰词 | 否。 {.text-danger} | | +| E091-07 | DISTINCT修饰词 | 是 {.text-success} | 并非所有聚合函数都支持该修饰词 | +| **E101** | **基本数据操作** | **部分**{.text-warning} | | +| E101-01 | INSERT（插入）语句 | 是 {.text-success} | 注：ClickHouse中的主键并不隐含`UNIQUE` 约束 | +| E101-03 | 可指定范围的UPDATE（更新）语句 | 部分 {.text-warning} | `ALTER UPDATE` 语句用来批量更新数据 | +| E101-04 | 可指定范围的DELETE（删除）语句 | 部分 {.text-warning} | `ALTER DELETE` 语句用来批量删除数据 | +| **E111** | **返回一行的SELECT语句** | **否**{.text-danger} | | +| **E121** | **基本游标支持** | **否**{.text-danger} | | +| E121-01 | DECLARE CURSOR | 否 {.text-danger} | | +| E121-02 | ORDER BY 涉及的列不需要出现在SELECT的列中 | 是 {.text-success} | | +| E121-03 | ORDER BY 从句中的表达式 | 是 {.text-success} | | +| E121-04 | OPEN 语句 | 否 {.text-danger} | | +| E121-06 | 受游标位置控制的 UPDATE 语句 | 否 {.text-danger} | | +| E121-07 | 受游标位置控制的 DELETE 语句 | 否 {.text-danger} | | +| E121-08 | CLOSE 语句 | 否 {.text-danger} | | +| E121-10 | FETCH 语句中包含隐式NEXT | 否 {.text-danger} | | +| E121-17 | WITH HOLD 游标 | 否 {.text-danger} | | +| **E131** | **空值支持** | **是**{.text-success} | 有部分限制 | +| **E141** | **基本完整性约束** | **部分**{.text-warning} | | +| E141-01 | NOT NULL（非空）约束 | 是 {.text-success} | 注: 默认情况下ClickHouse表中的列隐含`NOT NULL`约束 | +| E141-02 | NOT NULL（非空）列的UNIQUE（唯一）约束 | 否 {.text-danger} | | +| E141-03 | 主键约束 | 部分 {.text-warning} | | +| E141-04 | 对于引用删除和引用更新操作，基本的FOREIGN KEY（外键）约束默认不进行任何操作（NO ACTION） | 否 {.text-danger} | | +| E141-06 | CHECK（检查）约束 | 是 {.text-success} | | +| E141-07 | 列默认值 | 是 {.text-success} | | +| E141-08 | 在主键上推断非空 | 是 {.text-success} | | +| E141-10 | 可以按任何顺序指定外键中的名称 | 否 {.text-danger} | | +| **E151** | **事务支持** | **否**{.text-danger} | | +| E151-01 | COMMIT（提交）语句 | 否 {.text-danger} | | +| E151-02 | ROLLBACK（回滚）语句 | 否 {.text-danger} | | +| **E152** | **基本的SET TRANSACTION（设置事务隔离级别）语句** | **否**{.text-danger} | | +| E152-01 | SET TRANSACTION语句：ISOLATION LEVEL SERIALIZABLE（隔离级别为串行化）从句 | 否 {.text-danger} | | +| E152-02 | SET TRANSACTION语句：READ ONLY（只读）和READ WRITE（读写）从句 | 否 {.text-danger} | | +| **E153** | **具有子查询的可更新查询** | **是**{.text-success} | | +| **E161** | **使用“--”符号作为SQL注释** | **是**{.text-success} | | +| **E171** | **SQLSTATE支持** | **否**{.text-danger} | | +| **E182** | **主机语言绑定** | **否**{.text-danger} | | +| **F031** | **基本架构操作** | **部分**{.text-warning} | | +| F031-01 | 使用 CREATE TABLE 语句创建持久表 | 部分 {.text-warning} | 不支持 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 从句，不支持用户解析的数据类型 | +| F031-02 | CREATE VIEW（创建视图）语句 | 部分 {.text-warning} | 不支持 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 从句，不支持用户解析的数据类型 | +| F031-03 | GRANT（授权）语句 | 是 {.text-success} | | +| F031-04 | ALTER TABLE语句：ADD COLUMN从句 | 是 {.text-success} | 不支持 `GENERATED` 从句和以系统时间做参数 | +| F031-13 | DROP TABLE语句：RESTRICT从句 | 否 {.text-danger} | | +| F031-16 | DROP VIEW语句：RESTRICT子句 | 否 {.text-danger} | | +| F031-19 | REVOKE语句：RESTRICT子句 | 否 {.text-danger} | | +| **F041** | **基本连接关系** | **部分**{.text-warning} | | +| F041-01 | Inner join（但不一定是INNER关键字) | 是 {.text-success} | | +| F041-02 | INNER 关键字 | 是 {.text-success} | | +| F041-03 | LEFT OUTER JOIN | 是 {.text-success} | | +| F041-04 | RIGHT OUTER JOIN | 是 {.text-success} | | +| F041-05 | 外连接可嵌套 | 是 {.text-success} | | +| F041-07 | 左外部连接或右外连接中的内部表也可用于内部联接 | 是 {.text-success} | | +| F041-08 | 支持所有比较运算符（而不仅仅是=) | 否 {.text-danger} | | +| **F051** | **基本日期和时间** | **部分**{.text-warning} | | +| F051-01 | DATE（日期）数据类型（并支持用于表达日期的字面量) | 是 {.text-success} | | +| F051-02 | TIME（时间）数据类型（并支持用于表达时间的字面量），小数秒精度至少为0 | 否 {.text-danger} | | +| F051-03 | 时间戳数据类型（并支持用于表达时间戳的字面量），小数秒精度至少为0和6 | 是 {.text-danger} | | +| F051-04 | 日期、时间和时间戳数据类型的比较谓词 | 是 {.text-success} | | +| F051-05 | Datetime 类型和字符串形式表达的时间之间的显式转换 | 是 {.text-success} | | +| F051-06 | CURRENT_DATE | 否 {.text-danger} | 使用`today()`替代 | +| F051-07 | LOCALTIME | 否 {.text-danger} | 使用`now()`替代 | +| F051-08 | LOCALTIMESTAMP | 否 {.text-danger} | | +| **F081** | **视图的UNION和EXCEPT操作** | **部分**{.text-warning} | | +| **F131** | **分组操作** | **部分**{.text-warning} | | +| F131-01 | 在具有分组视图的查询中支持 WHERE、GROUP BY 和 HAVING 子句 | 是 {.text-success} | | +| F131-02 | 在分组视图中支持多张表 | 是 {.text-success} | | +| F131-03 | 分组视图的查询中支持集合函数 | 是 {.text-success} | | +| F131-04 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的子查询 | 是 {.text-success} | | +| F131-05 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的仅返回1条记录的SELECT查询 | 否 {.text-danger} | | +| **F181** | **多模块支持** | **否**{.text-danger} | | +| **F201** | **CAST 函数** | **是**{.text-success} | | +| **F221** | **显式默认值** | **否**{.text-danger} | | +| **F261** | **CASE 表达式** | **是**{.text-success} | | +| F261-01 | 简单 CASE 表达式 | 是 {.text-success} | | +| F261-02 | 搜索型 CASE 表达式 | 是 {.text-success} | | +| F261-03 | NULLIF | 是 {.text-success} | | +| F261-04 | COALESCE | 是 {.text-success} | | +| **F311** | **架构定义语句** | **部分**{.text-warning} | | +| F311-01 | CREATE SCHEMA | 部分 {.text-warning} | 见`CREATE DATABASE` | +| F311-02 | 用于创建持久表的 CREATE TABLE | 是 {.text-success} | | +| F311-03 | CREATE VIEW | 是 {.text-success} | | +| F311-04 | CREATE VIEW: WITH CHECK OPTION | 否 {.text-danger} | | +| F311-05 | GRANT 语句 | 是 {.text-success} | | +| **F471** | **标量子查询** | **是**{.text-success} | | +| **F481** | **扩展 NULL 谓词** | **是**{.text-success} | | +| **F812** | **基本标志位** | **否**{.text-danger} | +| **S011** | **用于不重复数据的数据类型** | **否**{.text-danger} | +| **T321** | **基本的SQL调用例程** | **否**{.text-danger} | | +| T321-01 | 没有重载的用户定义函数 | 否{.text-danger} | | +| T321-02 | 没有重载的用户定义存储过程 | 否{.text-danger} | | +| T321-03 | 功能调用 | 否 {.text-danger} | | +| T321-04 | CALL 语句 | 否 {.text-danger} | | +| T321-05 | RETURN 语句 | 否 {.text-danger} | | +| **T631** | **IN 谓词后接一个列表** | **是**{.text-success} | | diff --git a/docs/zh/sql-reference/data-types/int-uint.md b/docs/zh/sql-reference/data-types/int-uint.md index 3fb482639e7..e7fa27dcf70 100644 --- a/docs/zh/sql-reference/data-types/int-uint.md +++ b/docs/zh/sql-reference/data-types/int-uint.md @@ -1,17 +1,41 @@ -# UInt8,UInt16,UInt32,UInt64,Int8,Int16,Int32,Int64 {#uint8-uint16-uint32-uint64-int8-int16-int32-int64} +--- +toc_priority: 40 +toc_title: UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 +--- + +# UInt8, UInt16, UInt32, UInt64, UInt128, UInt256, Int8, Int16, Int32, Int64, Int128, Int256 固定长度的整型，包括有符号整型或无符号整型。 +创建表时，可以为整数设置类型参数 (例如. `TINYINT(8)`, `SMALLINT(16)`, `INT(32)`, `BIGINT(64)`), 但 ClickHouse 会忽略它们. + + ## 整型范围 {#int-ranges} -- Int8-\[-128:127\] -- Int16-\[-32768:32767\] -- Int32-\[-2147483648:2147483647\] -- Int64-\[-9223372036854775808:9223372036854775807\] + +- `Int8` — \[-128 : 127\] +- `Int16` — \[-32768 : 32767\] +- `Int32` — \[-2147483648 : 2147483647\] +- `Int64` — \[-9223372036854775808 : 9223372036854775807\] +- `Int128` — \[-170141183460469231731687303715884105728 : 170141183460469231731687303715884105727\] +- `Int256` — \[-57896044618658097711785492504343953926634992332820282019728792003956564819968 : 57896044618658097711785492504343953926634992332820282019728792003956564819967\] + +别名： + +- `Int8` — `TINYINT`, `BOOL`, `BOOLEAN`, `INT1`. +- `Int16` — `SMALLINT`, `INT2`. +- `Int32` — `INT`, `INT4`, `INTEGER`. +- `Int64` — `BIGINT`. ## 无符号整型范围 {#uint-ranges} -- UInt8-\[0:255\] -- UInt16-\[0:65535\] -- UInt32-\[0:4294967295\] -- UInt64-\[0:18446744073709551615\] + +- `UInt8` — \[0 : 255\] +- `UInt16` — \[0 : 65535\] +- `UInt32` — \[0 : 4294967295\] +- `UInt64` — \[0 : 18446744073709551615\] +- `UInt128` — \[0 : 340282366920938463463374607431768211455\] +- `UInt256` — \[0 : 115792089237316195423570985008687907853269984665640564039457584007913129639935\] + + +[源文档](https://clickhouse.com/docs/en/data_types/int_uint/) diff --git a/docs/zh/sql-reference/statements/alter/constraint.md b/docs/zh/sql-reference/statements/alter/constraint.md deleted file mode 120000 index e3b245408d1..00000000000 --- a/docs/zh/sql-reference/statements/alter/constraint.md +++ /dev/null @@ -1 +0,0 @@ -../../../../en/sql-reference/statements/alter/constraint.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/constraint.md b/docs/zh/sql-reference/statements/alter/constraint.md new file mode 100644 index 00000000000..9a71f0b055c --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/constraint.md @@ -0,0 +1,22 @@ +--- +toc_priority: 43 +toc_title: 约束 +--- + +# 操作约束 {#manipulations-with-constraints} + +约束可以使用以下语法添加或删除: + +``` sql +ALTER TABLE [db].name ADD CONSTRAINT constraint_name CHECK expression; +ALTER TABLE [db].name DROP CONSTRAINT constraint_name; +``` + +查看[constraints](../../../sql-reference/statements/create/table.md#constraints)。 + +查询将从表中添加或删除关于约束的元数据，因此它们将被立即处理。 + +!!! warning "警告" + 如果已有数据被添加，约束检查**将不会被执行**。 + +复制表上的所有更改都会被广播到ZooKeeper，并应用到其他副本上。 \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/index.md b/docs/zh/sql-reference/statements/alter/index.md index 2f60dbb262e..f7d983cab4e 100644 --- a/docs/zh/sql-reference/statements/alter/index.md +++ b/docs/zh/sql-reference/statements/alter/index.md @@ -1,23 +1,74 @@ --- -toc_hidden_folder: true -toc_priority: 42 -toc_title: INDEX +toc_priority: 35 +toc_title: ALTER --- -# 操作数据跳过索引 {#manipulations-with-data-skipping-indices} +## ALTER {#query_language_queries_alter} -可以使用以下操作： +大多数 `ALTER TABLE` 查询修改表设置或数据: -- `ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value [FIRST|AFTER name]` - 向表元数据添加索引描述。 +- [COLUMN](../../../sql-reference/statements/alter/column.md) +- [PARTITION](../../../sql-reference/statements/alter/partition.md) +- [DELETE](../../../sql-reference/statements/alter/delete.md) +- [UPDATE](../../../sql-reference/statements/alter/update.md) +- [ORDER BY](../../../sql-reference/statements/alter/order-by.md) +- [INDEX](../../../sql-reference/statements/alter/index/index.md) +- [CONSTRAINT](../../../sql-reference/statements/alter/constraint.md) +- [TTL](../../../sql-reference/statements/alter/ttl.md) -- `ALTER TABLE [db].name DROP INDEX name` - 从表元数据中删除索引描述并从磁盘中删除索引文件。 +!!! note "备注" + 大多数 `ALTER TABLE` 查询只支持[\*MergeTree](../../../engines/table-engines/mergetree-family/index.md)表，以及[Merge](../../../engines/table-engines/special/merge.md)和[Distributed](../../../engines/table-engines/special/distributed.md)。 -- `ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name` - 查询在分区`partition_name`中重建二级索引`name`。 操作为[mutation](../../../sql-reference/statements/alter/index.md#mutations). +这些 `ALTER` 语句操作视图: -前两个命令是轻量级的，它们只更改元数据或删除文件。 +- [ALTER TABLE ... MODIFY QUERY](../../../sql-reference/statements/alter/view.md) — 修改一个 [Materialized view](../create/view.md#materialized) 结构. +- [ALTER LIVE VIEW](../../../sql-reference/statements/alter/view.md#alter-live-view) — 刷新一个 [Live view](../create/view.md#live-view). -Also, they are replicated, syncing indices metadata via ZooKeeper. -此外，它们会被复制，会通过ZooKeeper同步索引元数据。 +这些 `ALTER` 语句修改与基于角色的访问控制相关的实体: -!!! note "注意" -索引操作仅支持具有以下特征的表 [`*MergeTree`](../../../engines/table-engines/mergetree-family/mergetree.md)引擎 (包括[replicated](../../../engines/table-engines/mergetree-family/replication.md)). +- [USER](../../../sql-reference/statements/alter/user.md) +- [ROLE](../../../sql-reference/statements/alter/role.md) +- [QUOTA](../../../sql-reference/statements/alter/quota.md) +- [ROW POLICY](../../../sql-reference/statements/alter/row-policy.md) +- [SETTINGS PROFILE](../../../sql-reference/statements/alter/settings-profile.md) + +[ALTER TABLE ... MODIFY COMMENT](../../../sql-reference/statements/alter/comment.md) 语句添加、修改或删除表中的注释，无论之前是否设置过。 + +## Mutations 突变 {#mutations} + +用来操作表数据的ALTER查询是通过一种叫做“突变”的机制来实现的，最明显的是[ALTER TABLE … DELETE](../../../sql-reference/statements/alter/delete.md)和[ALTER TABLE … UPDATE](../../../sql-reference/statements/alter/update.md)。它们是异步的后台进程，类似于[MergeTree](../../../engines/table-engines/mergetree-family/index.md)表的合并，产生新的“突变”版本的部件。 + + + +对于 `*MergeTree` 表，通过重写整个数据部分来执行突变。没有原子性——一旦突变的部件准备好，部件就会被替换，并且在突变期间开始执行的 `SELECT` 查询将看到来自已经突变的部件的数据，以及来自尚未突变的部件的数据。 + + + +突变完全按照它们的产生顺序排列，并按此顺序应用于每个部分。突变还与“INSERT INTO”查询进行部分排序:在提交突变之前插入表中的数据将被突变，而在此之后插入的数据将不会被突变。注意，突变不会以任何方式阻止插入。 + + + +突变查询在添加突变条目后立即返回(对于复制表到ZooKeeper，对于非复制表到文件系统)。突变本身使用系统配置文件设置异步执行。要跟踪突变的进程，可以使用[`system.mutations`](../../../operations/system-tables/mutations.md#system_tables-mutations) 表。成功提交的变异将继续执行，即使ClickHouse服务器重新启动。没有办法回滚突变一旦提交，但如果突变卡住了，它可以取消与[`KILL MUTATION`](../../../sql-reference/statements/misc.md#kill-mutation) 查询。 + + + +完成突变的条目不会立即删除(保留条目的数量由 `finished_mutations_to_keep` 存储引擎参数决定)。删除旧的突变条目。 + +## ALTER 查询的同步性 {#synchronicity-of-alter-queries} + + +对于非复制表，所有的 `ALTER` 查询都是同步执行的。对于复制表，查询只是向“ZooKeeper”添加相应动作的指令，动作本身会尽快执行。但是，查询可以等待所有副本上的这些操作完成。 + +对于所有的“ALTER”查询，您可以使用[replication_alter_partitions_sync](../../../operations/settings/settings.md#replication-alter-partitions-sync)设置等待。 + +通过[replication_wait_for_inactive_replica_timeout](../../../operations/settings/settings.md#replication-wait-for-inactive-replica-timeout]设置，可以指定不活动的副本执行所有 `ALTER` 查询的等待时间(以秒为单位)。 + + + +!!! info "备注" + + 对于所有的 `ALTER` 查询，如果 `replication_alter_partitions_sync = 2` 和一些副本的不激活时间超过时间(在 `replication_wait_for_inactive_replica_timeout` 设置中指定)，那么将抛出一个异常 `UNFINISHED`。 + + + +对于 `ALTER TABLE ... UPDATE|DELETE` 查询由 [mutations_sync](../../../operations/settings/settings.md#mutations_sync) 设置定义的同步度。 diff --git a/docs/zh/sql-reference/statements/alter/index/index.md b/docs/zh/sql-reference/statements/alter/index/index.md deleted file mode 120000 index b754fa71b83..00000000000 --- a/docs/zh/sql-reference/statements/alter/index/index.md +++ /dev/null @@ -1 +0,0 @@ -../../../../../en/sql-reference/statements/alter/index/index.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/alter/index/index.md b/docs/zh/sql-reference/statements/alter/index/index.md new file mode 100644 index 00000000000..16f48e55b2f --- /dev/null +++ b/docs/zh/sql-reference/statements/alter/index/index.md @@ -0,0 +1,23 @@ +--- +toc_hidden_folder: true +toc_priority: 42 +toc_title: INDEX +--- + +# 操作数据跳过索引 {#manipulations-with-data-skipping-indices} + +可以使用以下操作： + +- `ALTER TABLE [db].name ADD INDEX name expression TYPE type GRANULARITY value [FIRST|AFTER name]` - 向表元数据添加索引描述。 + +- `ALTER TABLE [db].name DROP INDEX name` - 从表元数据中删除索引描述并从磁盘中删除索引文件。 + +- `ALTER TABLE [db.]table MATERIALIZE INDEX name IN PARTITION partition_name` - 查询在分区`partition_name`中重建二级索引`name`。 操作为[mutation](../../../../sql-reference/statements/alter/index.md#mutations). + +前两个命令是轻量级的，它们只更改元数据或删除文件。 + +Also, they are replicated, syncing indices metadata via ZooKeeper. +此外，它们会被复制，会通过ZooKeeper同步索引元数据。 + +!!! note "注意" + 索引操作仅支持具有以下特征的表 [`*MergeTree`](../../../../engines/table-engines/mergetree-family/mergetree.md)引擎 (包括[replicated](../../../../engines/table-engines/mergetree-family/replication.md)). diff --git a/docs/zh/sql-reference/statements/exists.md b/docs/zh/sql-reference/statements/exists.md deleted file mode 120000 index d69e8224fe6..00000000000 --- a/docs/zh/sql-reference/statements/exists.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/exists.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/exists.md b/docs/zh/sql-reference/statements/exists.md new file mode 100644 index 00000000000..69b26fea918 --- /dev/null +++ b/docs/zh/sql-reference/statements/exists.md @@ -0,0 +1,12 @@ +--- +toc_priority: 45 +toc_title: EXISTS +--- + +# EXISTS 语句 {#exists-statement} + +``` sql +EXISTS [TEMPORARY] [TABLE|DICTIONARY] [db.]name [INTO OUTFILE filename] [FORMAT format] +``` + +返回一个单独的 `UInt8`类型的列，如果表或数据库不存在，则包含一个值 `0`，如果表在指定的数据库中存在，则包含一个值 `1`。 \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/set.md b/docs/zh/sql-reference/statements/set.md deleted file mode 120000 index 02e106afc9f..00000000000 --- a/docs/zh/sql-reference/statements/set.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/set.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/set.md b/docs/zh/sql-reference/statements/set.md new file mode 100644 index 00000000000..a9888a7080e --- /dev/null +++ b/docs/zh/sql-reference/statements/set.md @@ -0,0 +1,23 @@ +--- +toc_priority: 50 +toc_title: SET +--- + +# SET 语句 {#query-set} + +``` sql +SET param = value +``` + +给当前会话的 `param` [配置项](../../operations/settings/index.md)赋值。你不能用这样的方式修改[服务器相关设置](../../operations/server-configuration-parameters/index.md)。 + + +您还可以在单个查询中设置指定设置配置文件中的所有值。 + + + +``` sql +SET profile = 'profile-name-from-the-settings-file' +``` + +更多详情, 详见 [配置项](../../operations/settings/settings.md). diff --git a/docs/zh/sql-reference/statements/truncate.md b/docs/zh/sql-reference/statements/truncate.md deleted file mode 120000 index 92fbd705e8f..00000000000 --- a/docs/zh/sql-reference/statements/truncate.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/truncate.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/truncate.md b/docs/zh/sql-reference/statements/truncate.md new file mode 100644 index 00000000000..7d7bf7e47e2 --- /dev/null +++ b/docs/zh/sql-reference/statements/truncate.md @@ -0,0 +1,31 @@ +--- +toc_priority: 52 +toc_title: TRUNCATE +--- + +# TRUNCATE 语句 {#truncate-statement} + +``` sql +TRUNCATE TABLE [IF EXISTS] [db.]name [ON CLUSTER cluster] +``` + +删除表中的所有数据。当省略子句 `IF EXISTS` 时，如果表不存在，则查询返回一个错误。 + + + +`TRUNCATE` 查询不支持[View](../../engines/table-engines/special/view.md),[File](../../engines/table-engines/special/file.md), [URL](../../engines/table-engines/special/url.md), [Buffer](../../engines/table-engines/special/buffer.md) 和 [Null](../../engines/table-engines/special/null.md)表引擎。 + + + +可以使用 replication_alter_partitions_sync 设置在复制集上等待执行的操作。 + + + +通过 replication_wait_for_inactive_replica_timeout 设置，可以指定不活动副本执行 `TRUNCATE`查询需要等待多长时间(以秒为单位)。 + + + +!!! info "注意" + 如果`replication_alter_partitions_sync` 被设置为`2`，并且某些复制集超过 `replication_wait_for_inactive_replica_timeout`设置的时间不激活，那么将抛出一个异常`UNFINISHED`。 + + diff --git a/docs/zh/sql-reference/statements/use.md b/docs/zh/sql-reference/statements/use.md deleted file mode 120000 index 7bdbf049326..00000000000 --- a/docs/zh/sql-reference/statements/use.md +++ /dev/null @@ -1 +0,0 @@ -../../../en/sql-reference/statements/use.md \ No newline at end of file diff --git a/docs/zh/sql-reference/statements/use.md b/docs/zh/sql-reference/statements/use.md new file mode 100644 index 00000000000..41cba58bb9d --- /dev/null +++ b/docs/zh/sql-reference/statements/use.md @@ -0,0 +1,16 @@ +--- +toc_priority: 53 +toc_title: USE +--- + +# USE 语句 {#use} + +``` sql +USE db +``` + +用于设置会话的当前数据库。 + +如果查询语句中没有在表名前面以加点的方式指明数据库名， 则用当前数据库进行搜索。 + +使用 HTTP 协议时无法进行此查询，因为没有会话的概念。 diff --git a/docs/zh/whats-new/roadmap.md b/docs/zh/whats-new/roadmap.md index 3cb9dd6fa2f..8e8873c8ee4 100644 --- a/docs/zh/whats-new/roadmap.md +++ b/docs/zh/whats-new/roadmap.md @@ -5,6 +5,6 @@ toc_title: Roadmap # Roadmap {#roadmap} -`2021年Roadmap`已公布供公开讨论查看[这里](https://github.com/ClickHouse/ClickHouse/issues/17623). +`2022年Roadmap`已公布供公开讨论查看 [这里](https://github.com/ClickHouse/ClickHouse/issues/32513). {## [源文章](https://clickhouse.com/docs/en/roadmap/) ##} diff --git a/programs/client/Client.cpp b/programs/client/Client.cpp index a5e4517824d..2cbf39a61da 100644 --- a/programs/client/Client.cpp +++ b/programs/client/Client.cpp @@ -286,7 +286,7 @@ bool Client::executeMultiQuery(const String & all_queries_text) // , where the inline data is delimited by semicolon and not by a // newline. auto * insert_ast = parsed_query->as(); - if (insert_ast && insert_ast->data) + if (insert_ast && isSyncInsertWithData(*insert_ast, global_context)) { this_query_end = insert_ast->end; adjustQueryEnd(this_query_end, all_queries_end, global_context->getSettingsRef().max_parser_depth); @@ -481,48 +481,72 @@ catch (...) void Client::connect() { - connection_parameters = ConnectionParameters(config()); - - if (is_interactive) - std::cout << "Connecting to " - << (!connection_parameters.default_database.empty() ? "database " + connection_parameters.default_database + " at " - : "") - << connection_parameters.host << ":" << connection_parameters.port - << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "") << "." << std::endl; - String server_name; UInt64 server_version_major = 0; UInt64 server_version_minor = 0; UInt64 server_version_patch = 0; - try + for (size_t attempted_address_index = 0; attempted_address_index < hosts_and_ports.size(); ++attempted_address_index) { - connection = Connection::createConnection(connection_parameters, global_context); - - if (max_client_network_bandwidth) + try { - ThrottlerPtr throttler = std::make_shared(max_client_network_bandwidth, 0, ""); - connection->setThrottler(throttler); - } + connection_parameters + = ConnectionParameters(config(), hosts_and_ports[attempted_address_index].host, hosts_and_ports[attempted_address_index].port); - connection->getServerVersion( - connection_parameters.timeouts, server_name, server_version_major, server_version_minor, server_version_patch, server_revision); - } - catch (const Exception & e) - { - /// It is typical when users install ClickHouse, type some password and instantly forget it. - if ((connection_parameters.user.empty() || connection_parameters.user == "default") - && e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED) + if (is_interactive) + std::cout << "Connecting to " + << (!connection_parameters.default_database.empty() ? "database " + connection_parameters.default_database + " at " + : "") + << connection_parameters.host << ":" << connection_parameters.port + << (!connection_parameters.user.empty() ? " as user " + connection_parameters.user : "") << "." << std::endl; + + connection = Connection::createConnection(connection_parameters, global_context); + + if (max_client_network_bandwidth) + { + ThrottlerPtr throttler = std::make_shared(max_client_network_bandwidth, 0, ""); + connection->setThrottler(throttler); + } + + connection->getServerVersion( + connection_parameters.timeouts, server_name, server_version_major, server_version_minor, server_version_patch, server_revision); + config().setString("host", connection_parameters.host); + config().setInt("port", connection_parameters.port); + break; + } + catch (const Exception & e) { - std::cerr << std::endl - << "If you have installed ClickHouse and forgot password you can reset it in the configuration file." << std::endl - << "The password for default user is typically located at /etc/clickhouse-server/users.d/default-password.xml" << std::endl - << "and deleting this file will reset the password." << std::endl - << "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed." << std::endl - << std::endl; - } + /// It is typical when users install ClickHouse, type some password and instantly forget it. + /// This problem can't be fixed with reconnection so it is not attempted + if ((connection_parameters.user.empty() || connection_parameters.user == "default") + && e.code() == DB::ErrorCodes::AUTHENTICATION_FAILED) + { + std::cerr << std::endl + << "If you have installed ClickHouse and forgot password you can reset it in the configuration file." << std::endl + << "The password for default user is typically located at /etc/clickhouse-server/users.d/default-password.xml" << std::endl + << "and deleting this file will reset the password." << std::endl + << "See also /etc/clickhouse-server/users.xml on the server where ClickHouse is installed." << std::endl + << std::endl; + throw; + } + else + { + if (attempted_address_index == hosts_and_ports.size() - 1) + throw; - throw; + if (is_interactive) + { + std::cerr << "Connection attempt to database at " + << connection_parameters.host << ":" << connection_parameters.port + << " resulted in failure" + << std::endl + << getExceptionMessage(e, false) + << std::endl + << "Attempting connection to the next provided address" + << std::endl; + } + } + } } server_version = toString(server_version_major) + "." + toString(server_version_minor) + "." + toString(server_version_patch); @@ -966,8 +990,6 @@ void Client::addOptions(OptionsDescription & options_description) /// Main commandline options related to client functionality and all parameters from Settings. options_description.main_description->add_options() ("config,c", po::value(), "config-file path (another shorthand)") - ("host,h", po::value()->default_value("localhost"), "server host") - ("port", po::value()->default_value(9000), "server port") ("secure,s", "Use TLS connection") ("user,u", po::value()->default_value("default"), "user") /** If "--password [value]" is used but the value is omitted, the bad argument exception will be thrown. @@ -1013,12 +1035,24 @@ void Client::addOptions(OptionsDescription & options_description) ( "types", po::value(), "types" ); + + /// Commandline options related to hosts and ports. + options_description.hosts_and_ports_description.emplace(createOptionsDescription("Hosts and ports options", terminal_width)); + options_description.hosts_and_ports_description->add_options() + ("host,h", po::value()->default_value("localhost"), + "Server hostname. Multiple hosts can be passed via multiple arguments" + "Example of usage: '--host host1 --host host2 --port port2 --host host3 ...'" + "Each '--port port' will be attached to the last seen host that doesn't have a port yet," + "if there is no such host, the port will be attached to the next first host or to default host.") + ("port", po::value()->default_value(DBMS_DEFAULT_PORT), "server ports") + ; } void Client::processOptions(const OptionsDescription & options_description, const CommandLineOptions & options, - const std::vector & external_tables_arguments) + const std::vector & external_tables_arguments, + const std::vector & hosts_and_ports_arguments) { namespace po = boost::program_options; @@ -1050,6 +1084,25 @@ void Client::processOptions(const OptionsDescription & options_description, exit(exit_code); } } + + if (hosts_and_ports_arguments.empty()) + { + hosts_and_ports.emplace_back(HostAndPort{"localhost", DBMS_DEFAULT_PORT}); + } + else + { + for (const auto & hosts_and_ports_argument : hosts_and_ports_arguments) + { + /// Parse commandline options related to external tables. + po::parsed_options parsed_hosts_and_ports + = po::command_line_parser(hosts_and_ports_argument).options(options_description.hosts_and_ports_description.value()).run(); + po::variables_map host_and_port_options; + po::store(parsed_hosts_and_ports, host_and_port_options); + hosts_and_ports.emplace_back( + HostAndPort{host_and_port_options["host"].as(), host_and_port_options["port"].as()}); + } + } + send_external_tables = true; shared_context = Context::createShared(); @@ -1074,12 +1127,8 @@ void Client::processOptions(const OptionsDescription & options_description, if (options.count("config")) config().setString("config-file", options["config"].as()); - if (options.count("host") && !options["host"].defaulted()) - config().setString("host", options["host"].as()); if (options.count("interleave-queries-file")) interleave_queries_files = options["interleave-queries-file"].as>(); - if (options.count("port") && !options["port"].defaulted()) - config().setInt("port", options["port"].as()); if (options.count("secure")) config().setBool("secure", true); if (options.count("user") && !options["user"].defaulted()) diff --git a/programs/client/Client.h b/programs/client/Client.h index 2def74ef3fc..45bdd077351 100644 --- a/programs/client/Client.h +++ b/programs/client/Client.h @@ -25,8 +25,11 @@ protected: void printHelpMessage(const OptionsDescription & options_description) override; void addOptions(OptionsDescription & options_description) override; - void processOptions(const OptionsDescription & options_description, const CommandLineOptions & options, - const std::vector & external_tables_arguments) override; + void processOptions( + const OptionsDescription & options_description, + const CommandLineOptions & options, + const std::vector & external_tables_arguments, + const std::vector & hosts_and_ports_arguments) override; void processConfig() override; private: diff --git a/programs/format/Format.cpp b/programs/format/Format.cpp index 4b0e8ad1ca1..835afcdb2ed 100644 --- a/programs/format/Format.cpp +++ b/programs/format/Format.cpp @@ -57,8 +57,16 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ("seed", po::value(), "seed (arbitrary string) that determines the result of obfuscation") ; + Settings cmd_settings; + for (const auto & field : cmd_settings.all()) + { + if (field.getName() == "max_parser_depth" || field.getName() == "max_query_size") + cmd_settings.addProgramOption(desc, field); + } + boost::program_options::variables_map options; boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); + po::notify(options); if (options.count("help")) { @@ -149,7 +157,8 @@ int mainEntryClickHouseFormat(int argc, char ** argv) ParserQuery parser(end); do { - ASTPtr res = parseQueryAndMovePosition(parser, pos, end, "query", multiple, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + ASTPtr res = parseQueryAndMovePosition( + parser, pos, end, "query", multiple, cmd_settings.max_query_size, cmd_settings.max_parser_depth); /// For insert query with data(INSERT INTO ... VALUES ...), will lead to format fail, /// should throw exception early and make exception message more readable. if (const auto * insert_query = res->as(); insert_query && insert_query->data) @@ -222,6 +231,5 @@ int mainEntryClickHouseFormat(int argc, char ** argv) std::cerr << getCurrentExceptionMessage(true) << '\n'; return getCurrentExceptionCode(); } - return 0; } diff --git a/programs/keeper/Keeper.cpp b/programs/keeper/Keeper.cpp index 636ce129d63..88df4d5b3e7 100644 --- a/programs/keeper/Keeper.cpp +++ b/programs/keeper/Keeper.cpp @@ -324,7 +324,7 @@ int Keeper::main(const std::vector & /*args*/) } else { - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); } } diff --git a/programs/library-bridge/Handlers.cpp b/programs/library-bridge/Handlers.cpp index bf9ace679ba..58f9bd0a936 100644 --- a/programs/library-bridge/Handlers.cpp +++ b/programs/library-bridge/Handlers.cpp @@ -37,7 +37,7 @@ namespace if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(&Poco::Logger::get("LibraryBridge"), message); + LOG_WARNING(&Poco::Logger::get("LibraryBridge"), fmt::runtime(message)); } std::shared_ptr parseColumns(std::string && column_string) @@ -123,7 +123,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe } else { - LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead"); + LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead", from_dictionary_id); lib_new = true; } } @@ -178,7 +178,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe catch (const Exception & ex) { processError(response, "Invalid 'sample_block' parameter in request body '" + ex.message() + "'"); - LOG_WARNING(log, ex.getStackTraceString()); + LOG_WARNING(log, fmt::runtime(ex.getStackTraceString())); return; } @@ -278,7 +278,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe catch (const Exception & ex) { processError(response, "Invalid 'requested_block' parameter in request body '" + ex.message() + "'"); - LOG_WARNING(log, ex.getStackTraceString()); + LOG_WARNING(log, fmt::runtime(ex.getStackTraceString())); return; } diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index ce39f22e978..eb3a03d0564 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -313,9 +313,15 @@ void LocalServer::cleanup() } +static bool checkIfStdinIsRegularFile() +{ + struct stat file_stat; + return fstat(STDIN_FILENO, &file_stat) == 0 && S_ISREG(file_stat.st_mode); +} + std::string LocalServer::getInitialCreateTableQuery() { - if (!config().has("table-structure") && !config().has("table-file")) + if (!config().has("table-structure") && !config().has("table-file") && !config().has("table-data-format") && (!checkIfStdinIsRegularFile() || !config().has("query"))) return {}; auto table_name = backQuoteIfNeed(config().getString("table-name", "table")); @@ -327,6 +333,7 @@ std::string LocalServer::getInitialCreateTableQuery() { /// Use Unix tools stdin naming convention table_file = "stdin"; + format_from_file_name = FormatFactory::instance().getFormatFromFileDescriptor(STDIN_FILENO); } else { @@ -336,8 +343,9 @@ std::string LocalServer::getInitialCreateTableQuery() format_from_file_name = FormatFactory::instance().getFormatFromFileName(file_name, false); } - auto data_format - = backQuoteIfNeed(config().getString("table-data-format", format_from_file_name.empty() ? "TSV" : format_from_file_name)); + auto data_format = backQuoteIfNeed( + config().getString("table-data-format", config().getString("format", format_from_file_name.empty() ? "TSV" : format_from_file_name))); + if (table_structure == "auto") table_structure = ""; @@ -517,22 +525,17 @@ void LocalServer::processConfig() if (config().has("multiquery")) is_multiquery = true; - - load_suggestions = true; } else { - if (delayed_interactive) - { - load_suggestions = true; - } - need_render_progress = config().getBool("progress", false); echo_queries = config().hasOption("echo") || config().hasOption("verbose"); ignore_error = config().getBool("ignore-error", false); is_multiquery = true; } + print_stack_trace = config().getBool("stacktrace", false); + load_suggestions = (is_interactive || delayed_interactive) && !config().getBool("disable_suggestion", false); auto logging = (config().has("logger.console") || config().has("logger.level") @@ -772,7 +775,7 @@ void LocalServer::applyCmdOptions(ContextMutablePtr context) } -void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector &) +void LocalServer::processOptions(const OptionsDescription &, const CommandLineOptions & options, const std::vector &, const std::vector &) { if (options.count("table")) config().setString("table-name", options["table"].as()); diff --git a/programs/local/LocalServer.h b/programs/local/LocalServer.h index 06e3746eacd..cc186e343c1 100644 --- a/programs/local/LocalServer.h +++ b/programs/local/LocalServer.h @@ -41,7 +41,7 @@ protected: void addOptions(OptionsDescription & options_description) override; void processOptions(const OptionsDescription & options_description, const CommandLineOptions & options, - const std::vector &) override; + const std::vector &, const std::vector &) override; void processConfig() override; private: diff --git a/programs/odbc-bridge/ColumnInfoHandler.cpp b/programs/odbc-bridge/ColumnInfoHandler.cpp index 8ceeddcd7ab..4d9a6b7a692 100644 --- a/programs/odbc-bridge/ColumnInfoHandler.cpp +++ b/programs/odbc-bridge/ColumnInfoHandler.cpp @@ -77,7 +77,7 @@ void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServ response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); }; if (!params.has("table")) diff --git a/programs/odbc-bridge/IdentifierQuoteHandler.cpp b/programs/odbc-bridge/IdentifierQuoteHandler.cpp index c7cad68f19e..7f809da4b10 100644 --- a/programs/odbc-bridge/IdentifierQuoteHandler.cpp +++ b/programs/odbc-bridge/IdentifierQuoteHandler.cpp @@ -29,7 +29,7 @@ void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServ response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); }; if (!params.has("connection_string")) diff --git a/programs/odbc-bridge/MainHandler.cpp b/programs/odbc-bridge/MainHandler.cpp index 1252d1ae70a..02bdabe8ffa 100644 --- a/programs/odbc-bridge/MainHandler.cpp +++ b/programs/odbc-bridge/MainHandler.cpp @@ -46,7 +46,7 @@ void ODBCHandler::processError(HTTPServerResponse & response, const std::string response.setStatusAndReason(HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); } @@ -102,7 +102,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse catch (const Exception & ex) { processError(response, "Invalid 'sample_block' parameter in request body '" + ex.message() + "'"); - LOG_ERROR(log, ex.getStackTraceString()); + LOG_ERROR(log, fmt::runtime(ex.getStackTraceString())); return; } diff --git a/programs/odbc-bridge/SchemaAllowedHandler.cpp b/programs/odbc-bridge/SchemaAllowedHandler.cpp index 7b526bd8041..0c58af2f7c1 100644 --- a/programs/odbc-bridge/SchemaAllowedHandler.cpp +++ b/programs/odbc-bridge/SchemaAllowedHandler.cpp @@ -37,7 +37,7 @@ void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServer response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR); if (!response.sent()) *response.send() << message << std::endl; - LOG_WARNING(log, message); + LOG_WARNING(log, fmt::runtime(message)); }; if (!params.has("connection_string")) diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 98838cb2337..79837310ec4 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -80,6 +81,7 @@ #include #include #include +#include #include #include #include @@ -194,6 +196,7 @@ namespace { void setupTmpPath(Poco::Logger * log, const std::string & path) +try { LOG_DEBUG(log, "Setting up {} to store temporary data in it", path); @@ -212,6 +215,15 @@ void setupTmpPath(Poco::Logger * log, const std::string & path) LOG_DEBUG(log, "Skipped file in temporary path {}", it->path().string()); } } +catch (...) +{ + DB::tryLogCurrentException( + log, + fmt::format( + "Caught exception while setup temporary path: {}. It is ok to skip this exception as cleaning old temporary files is not " + "necessary", + path)); +} int waitServersToFinish(std::vector & servers, size_t seconds_to_wait) { @@ -914,6 +926,14 @@ if (ThreadFuzzer::instance().isEffective()) total_memory_tracker.setDescription("(total)"); total_memory_tracker.setMetric(CurrentMetrics::MemoryTracking); + auto * global_overcommit_tracker = global_context->getGlobalOvercommitTracker(); + if (config->has("global_memory_usage_overcommit_max_wait_microseconds")) + { + UInt64 max_overcommit_wait_time = config->getUInt64("global_memory_usage_overcommit_max_wait_microseconds", 0); + global_overcommit_tracker->setMaxWaitTime(max_overcommit_wait_time); + } + total_memory_tracker.setOvercommitTracker(global_overcommit_tracker); + // FIXME logging-related things need synchronization -- see the 'Logger * log' saved // in a lot of places. For now, disable updating log configuration without server restart. //setTextLog(global_context->getTextLog()); @@ -962,7 +982,9 @@ if (ThreadFuzzer::instance().isEffective()) global_context->updateInterserverCredentials(*config); CompressionCodecEncrypted::Configuration::instance().tryLoad(*config, "encryption_codecs"); - +#if USE_SSL + CertificateReloader::instance().tryLoad(*config); +#endif ProfileEvents::increment(ProfileEvents::MainConfigLoads); }, /* already_loaded = */ false); /// Reload it right now (initial loading) @@ -1352,6 +1374,16 @@ if (ThreadFuzzer::instance().isEffective()) ErrorCodes::NO_ELEMENTS_IN_CONFIG); } + if (servers.empty()) + throw Exception("No servers started (add valid listen_host and 'tcp_port' or 'http_port' to configuration file.)", + ErrorCodes::NO_ELEMENTS_IN_CONFIG); + +#if USE_SSL + CertificateReloader::instance().tryLoad(config()); +#endif + + /// Must be done after initialization of `servers`, because async_metrics will access `servers` variable from its thread. + async_metrics.start(); { diff --git a/programs/server/config.xml b/programs/server/config.xml index d88773a3fc4..def64607caf 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -217,13 +217,12 @@ /path/to/ssl_ca_cert_file - - deflate + none - - medium + + 0 -1 @@ -244,7 +243,7 @@ openssl dhparam -out /etc/clickhouse-server/dhparam.pem 4096 Only file format with BEGIN DH PARAMETERS is supported. --> - /etc/clickhouse-server/dhparam.pem + none true true diff --git a/programs/static-files-disk-uploader/static-files-disk-uploader.cpp b/programs/static-files-disk-uploader/static-files-disk-uploader.cpp index ad3a3090de6..6ce84f823e9 100644 --- a/programs/static-files-disk-uploader/static-files-disk-uploader.cpp +++ b/programs/static-files-disk-uploader/static-files-disk-uploader.cpp @@ -31,7 +31,7 @@ namespace ErrorCodes * If test-mode option is added, files will be put by given url via PUT request. */ -void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, WriteBuffer & metadata_buf) +void processFile(const fs::path & file_path, const fs::path & dst_path, bool test_mode, bool link, WriteBuffer & metadata_buf) { String remote_path; RE2::FullMatch(file_path.string(), EXTRACT_PATH_PATTERN, &remote_path); @@ -52,22 +52,29 @@ void processFile(const fs::path & file_path, const fs::path & dst_path, bool tes auto dst_file_path = fs::path(dst_path) / remote_path; - auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path)); - std::shared_ptr dst_buf; - - /// test mode for integration tests. - if (test_mode) - dst_buf = std::make_shared(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT); + if (link) + { + fs::create_symlink(file_path, dst_file_path); + } else - dst_buf = std::make_shared(dst_file_path); + { + auto src_buf = createReadBufferFromFileBase(file_path, {}, fs::file_size(file_path)); + std::shared_ptr dst_buf; - copyData(*src_buf, *dst_buf); - dst_buf->next(); - dst_buf->finalize(); + /// test mode for integration tests. + if (test_mode) + dst_buf = std::make_shared(Poco::URI(dst_file_path), Poco::Net::HTTPRequest::HTTP_PUT); + else + dst_buf = std::make_shared(dst_file_path); + + copyData(*src_buf, *dst_buf); + dst_buf->next(); + dst_buf->finalize(); + } }; -void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode) +void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_mode, bool link) { std::cerr << "Data path: " << data_path << ", destination path: " << dst_path << std::endl; @@ -94,7 +101,7 @@ void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_ { if (dir_it->is_directory()) { - processFile(dir_it->path(), dst_path, test_mode, *root_meta); + processFile(dir_it->path(), dst_path, test_mode, link, *root_meta); String directory_prefix; RE2::FullMatch(dir_it->path().string(), EXTRACT_PATH_PATTERN, &directory_prefix); @@ -115,14 +122,14 @@ void processTableFiles(const fs::path & data_path, fs::path dst_path, bool test_ fs::directory_iterator files_end; for (fs::directory_iterator file_it(dir_it->path()); file_it != files_end; ++file_it) - processFile(file_it->path(), dst_path, test_mode, *directory_meta); + processFile(file_it->path(), dst_path, test_mode, link, *directory_meta); directory_meta->next(); directory_meta->finalize(); } else { - processFile(dir_it->path(), dst_path, test_mode, *root_meta); + processFile(dir_it->path(), dst_path, test_mode, link, *root_meta); } } root_meta->next(); @@ -141,6 +148,7 @@ try ("help,h", "produce help message") ("metadata-path", po::value(), "Metadata path (select data_paths from system.tables where name='table_name'") ("test-mode", "Use test mode, which will put data on given url via PUT") + ("link", "Create symlinks instead of copying") ("url", po::value(), "Web server url for test mode") ("output-dir", po::value(), "Directory to put files in non-test mode"); @@ -186,7 +194,7 @@ try root_path = fs::current_path(); } - processTableFiles(fs_path, root_path, test_mode); + processTableFiles(fs_path, root_path, test_mode, options.count("link")); return 0; } diff --git a/src/Access/AccessRights.cpp b/src/Access/AccessRights.cpp index 19b069546ee..ca8609f3984 100644 --- a/src/Access/AccessRights.cpp +++ b/src/Access/AccessRights.cpp @@ -1,8 +1,8 @@ #include #include +#include #include #include -#include #include namespace DB @@ -101,7 +101,7 @@ namespace AccessRightsElements getResult() const { ProtoElements sorted = *this; - boost::range::sort(sorted); + ::sort(sorted.begin(), sorted.end()); AccessRightsElements res; res.reserve(sorted.size()); diff --git a/src/Access/CachedAccessChecking.cpp b/src/Access/CachedAccessChecking.cpp new file mode 100644 index 00000000000..aa8ef6073d3 --- /dev/null +++ b/src/Access/CachedAccessChecking.cpp @@ -0,0 +1,44 @@ +#include +#include + + +namespace DB +{ +CachedAccessChecking::CachedAccessChecking(const std::shared_ptr & access_, AccessFlags access_flags_) + : CachedAccessChecking(access_, AccessRightsElement{access_flags_}) +{ +} + +CachedAccessChecking::CachedAccessChecking(const std::shared_ptr & access_, const AccessRightsElement & element_) + : access(access_), element(element_) +{ +} + +CachedAccessChecking::~CachedAccessChecking() = default; + +bool CachedAccessChecking::checkAccess(bool throw_if_denied) +{ + if (checked) + return result; + if (throw_if_denied) + { + try + { + access->checkAccess(element); + result = true; + } + catch (...) + { + result = false; + throw; + } + } + else + { + result = access->isGranted(element); + } + checked = true; + return result; +} + +} diff --git a/src/Access/CachedAccessChecking.h b/src/Access/CachedAccessChecking.h new file mode 100644 index 00000000000..e87c28dd823 --- /dev/null +++ b/src/Access/CachedAccessChecking.h @@ -0,0 +1,29 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class ContextAccess; + +/// Checks if the current user has a specified access type granted, +/// and if it's checked another time later, it will just return the first result. +class CachedAccessChecking +{ +public: + CachedAccessChecking(const std::shared_ptr & access_, AccessFlags access_flags_); + CachedAccessChecking(const std::shared_ptr & access_, const AccessRightsElement & element_); + ~CachedAccessChecking(); + + bool checkAccess(bool throw_if_denied = true); + +private: + const std::shared_ptr access; + const AccessRightsElement element; + bool checked = false; + bool result = false; +}; + +} diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index 0ccf5e85624..0b69bd5fd0e 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -86,7 +86,7 @@ enum class AccessType M(CREATE_DICTIONARY, "", DICTIONARY, CREATE) /* allows to execute {CREATE|ATTACH} DICTIONARY */\ M(CREATE_TEMPORARY_TABLE, "", GLOBAL, CREATE) /* allows to create and manipulate temporary tables; implicitly enabled by the grant CREATE_TABLE on any table */ \ - M(CREATE_FUNCTION, "", DATABASE, CREATE) /* allows to execute CREATE FUNCTION */ \ + M(CREATE_FUNCTION, "", GLOBAL, CREATE) /* allows to execute CREATE FUNCTION */ \ M(CREATE, "", GROUP, ALL) /* allows to execute {CREATE|ATTACH} */ \ \ M(DROP_DATABASE, "", DATABASE, DROP) /* allows to execute {DROP|DETACH} DATABASE */\ @@ -94,7 +94,7 @@ enum class AccessType M(DROP_VIEW, "", VIEW, DROP) /* allows to execute {DROP|DETACH} TABLE for views; implicitly enabled by the grant DROP_TABLE */\ M(DROP_DICTIONARY, "", DICTIONARY, DROP) /* allows to execute {DROP|DETACH} DICTIONARY */\ - M(DROP_FUNCTION, "", DATABASE, DROP) /* allows to execute DROP FUNCTION */\ + M(DROP_FUNCTION, "", GLOBAL, DROP) /* allows to execute DROP FUNCTION */\ M(DROP, "", GROUP, ALL) /* allows to execute {DROP|DETACH} */\ \ M(TRUNCATE, "TRUNCATE TABLE", TABLE, ALL) \ @@ -113,9 +113,9 @@ enum class AccessType M(ALTER_ROLE, "", GLOBAL, ACCESS_MANAGEMENT) \ M(DROP_ROLE, "", GLOBAL, ACCESS_MANAGEMENT) \ M(ROLE_ADMIN, "", GLOBAL, ACCESS_MANAGEMENT) /* allows to grant and revoke the roles which are not granted to the current user with admin option */\ - M(CREATE_ROW_POLICY, "CREATE POLICY", GLOBAL, ACCESS_MANAGEMENT) \ - M(ALTER_ROW_POLICY, "ALTER POLICY", GLOBAL, ACCESS_MANAGEMENT) \ - M(DROP_ROW_POLICY, "DROP POLICY", GLOBAL, ACCESS_MANAGEMENT) \ + M(CREATE_ROW_POLICY, "CREATE POLICY", TABLE, ACCESS_MANAGEMENT) \ + M(ALTER_ROW_POLICY, "ALTER POLICY", TABLE, ACCESS_MANAGEMENT) \ + M(DROP_ROW_POLICY, "DROP POLICY", TABLE, ACCESS_MANAGEMENT) \ M(CREATE_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \ M(ALTER_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \ M(DROP_QUOTA, "", GLOBAL, ACCESS_MANAGEMENT) \ @@ -124,7 +124,7 @@ enum class AccessType M(DROP_SETTINGS_PROFILE, "DROP PROFILE", GLOBAL, ACCESS_MANAGEMENT) \ M(SHOW_USERS, "SHOW CREATE USER", GLOBAL, SHOW_ACCESS) \ M(SHOW_ROLES, "SHOW CREATE ROLE", GLOBAL, SHOW_ACCESS) \ - M(SHOW_ROW_POLICIES, "SHOW POLICIES, SHOW CREATE ROW POLICY, SHOW CREATE POLICY", GLOBAL, SHOW_ACCESS) \ + M(SHOW_ROW_POLICIES, "SHOW POLICIES, SHOW CREATE ROW POLICY, SHOW CREATE POLICY", TABLE, SHOW_ACCESS) \ M(SHOW_QUOTAS, "SHOW CREATE QUOTA", GLOBAL, SHOW_ACCESS) \ M(SHOW_SETTINGS_PROFILES, "SHOW PROFILES, SHOW CREATE SETTINGS PROFILE, SHOW CREATE PROFILE", GLOBAL, SHOW_ACCESS) \ M(SHOW_ACCESS, "", GROUP, ACCESS_MANAGEMENT) \ @@ -166,6 +166,7 @@ enum class AccessType M(dictGet, "dictHas, dictGetHierarchy, dictIsIn", DICTIONARY, ALL) /* allows to execute functions dictGet(), dictHas(), dictGetHierarchy(), dictIsIn() */\ \ M(addressToLine, "", GLOBAL, INTROSPECTION) /* allows to execute function addressToLine() */\ + M(addressToLineWithInlines, "", GLOBAL, INTROSPECTION) /* allows to execute function addressToLineWithInlines() */\ M(addressToSymbol, "", GLOBAL, INTROSPECTION) /* allows to execute function addressToSymbol() */\ M(demangle, "", GLOBAL, INTROSPECTION) /* allows to execute function demangle() */\ M(INTROSPECTION, "INTROSPECTION FUNCTIONS", GROUP, ALL) /* allows to execute functions addressToLine(), addressToSymbol(), demangle()*/\ diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 400ee55a35d..744c3571175 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -425,6 +425,7 @@ bool ContextAccess::checkAccessImplHelper(const AccessFlags & flags, const Args | AccessType::TRUNCATE; const AccessFlags dictionary_ddl = AccessType::CREATE_DICTIONARY | AccessType::DROP_DICTIONARY; + const AccessFlags function_ddl = AccessType::CREATE_FUNCTION | AccessType::DROP_FUNCTION; const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl; const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE; const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS; @@ -432,7 +433,7 @@ bool ContextAccess::checkAccessImplHelper(const AccessFlags & flags, const Args const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY; const AccessFlags not_readonly_1_flags = AccessType::CREATE_TEMPORARY_TABLE; - const AccessFlags ddl_flags = table_ddl | dictionary_ddl; + const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl; const AccessFlags introspection_flags = AccessType::INTROSPECTION; }; static const PrecalculatedFlags precalc; diff --git a/src/Access/RolesOrUsersSet.cpp b/src/Access/RolesOrUsersSet.cpp index 810198eeb98..2c302fde229 100644 --- a/src/Access/RolesOrUsersSet.cpp +++ b/src/Access/RolesOrUsersSet.cpp @@ -7,8 +7,8 @@ #include #include #include -#include #include +#include namespace DB @@ -132,7 +132,7 @@ std::shared_ptr RolesOrUsersSet::toAST() const ast->names.reserve(ids.size()); for (const UUID & id : ids) ast->names.emplace_back(::DB::toString(id)); - boost::range::sort(ast->names); + ::sort(ast->names.begin(), ast->names.end()); } if (!except_ids.empty()) @@ -140,7 +140,7 @@ std::shared_ptr RolesOrUsersSet::toAST() const ast->except_names.reserve(except_ids.size()); for (const UUID & except_id : except_ids) ast->except_names.emplace_back(::DB::toString(except_id)); - boost::range::sort(ast->except_names); + ::sort(ast->except_names.begin(), ast->except_names.end()); } return ast; @@ -161,7 +161,7 @@ std::shared_ptr RolesOrUsersSet::toASTWithNames(const Access if (name) ast->names.emplace_back(std::move(*name)); } - boost::range::sort(ast->names); + ::sort(ast->names.begin(), ast->names.end()); } if (!except_ids.empty()) @@ -173,7 +173,7 @@ std::shared_ptr RolesOrUsersSet::toASTWithNames(const Access if (except_name) ast->except_names.emplace_back(std::move(*except_name)); } - boost::range::sort(ast->except_names); + ::sort(ast->except_names.begin(), ast->except_names.end()); } return ast; diff --git a/src/Access/tests/gtest_access_rights_ops.cpp b/src/Access/tests/gtest_access_rights_ops.cpp index 2881825dd17..3f3abff4e87 100644 --- a/src/Access/tests/gtest_access_rights_ops.cpp +++ b/src/Access/tests/gtest_access_rights_ops.cpp @@ -45,7 +45,15 @@ TEST(AccessRights, Union) lhs.grant(AccessType::INSERT); rhs.grant(AccessType::ALL, "db1"); lhs.makeUnion(rhs); - ASSERT_EQ(lhs.toString(), "GRANT INSERT ON *.*, GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, CREATE FUNCTION, DROP, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*"); + ASSERT_EQ(lhs.toString(), + "GRANT INSERT ON *.*, " + "GRANT SHOW, SELECT, ALTER, CREATE DATABASE, CREATE TABLE, CREATE VIEW, " + "CREATE DICTIONARY, DROP DATABASE, DROP TABLE, DROP VIEW, DROP DICTIONARY, " + "TRUNCATE, OPTIMIZE, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, " + "SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, " + "SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, " + "SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, " + "SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON db1.*"); } diff --git a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h index eee91904b9b..532e1ce50b3 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h +++ b/src/AggregateFunctions/AggregateFunctionGroupBitmapData.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -557,7 +558,7 @@ public: } if (limit < answer.size()) { - std::nth_element(answer.begin(), answer.begin() + limit, answer.end()); + ::nth_element(answer.begin(), answer.begin() + limit, answer.end()); answer.resize(limit); } diff --git a/src/AggregateFunctions/AggregateFunctionHistogram.h b/src/AggregateFunctions/AggregateFunctionHistogram.h index b858c6b628c..047ddf1ddeb 100644 --- a/src/AggregateFunctions/AggregateFunctionHistogram.h +++ b/src/AggregateFunctions/AggregateFunctionHistogram.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -72,7 +74,7 @@ private: private: void sort() { - std::sort(points, points + size, + ::sort(points, points + size, [](const WeightedValue & first, const WeightedValue & second) { return first.mean < second.mean; diff --git a/src/AggregateFunctions/AggregateFunctionIf.cpp b/src/AggregateFunctions/AggregateFunctionIf.cpp index 5ba54ff8505..ce71e76de43 100644 --- a/src/AggregateFunctions/AggregateFunctionIf.cpp +++ b/src/AggregateFunctions/AggregateFunctionIf.cpp @@ -72,7 +72,7 @@ private: using Base = AggregateFunctionNullBase>; - inline bool singleFilter(const IColumn ** columns, size_t row_num, size_t num_arguments) const + inline bool singleFilter(const IColumn ** columns, size_t row_num) const { const IColumn * filter_column = columns[num_arguments - 1]; @@ -112,7 +112,7 @@ public: { const ColumnNullable * column = assert_cast(columns[0]); const IColumn * nested_column = &column->getNestedColumn(); - if (!column->isNullAt(row_num) && singleFilter(columns, row_num, num_arguments)) + if (!column->isNullAt(row_num) && singleFilter(columns, row_num)) { this->setFlag(place); this->nested_function->add(this->nestedPlace(place), &nested_column, row_num, arena); diff --git a/src/AggregateFunctions/AggregateFunctionIntervalLengthSum.h b/src/AggregateFunctions/AggregateFunctionIntervalLengthSum.h index 443d76f47cb..92f527f7c43 100644 --- a/src/AggregateFunctions/AggregateFunctionIntervalLengthSum.h +++ b/src/AggregateFunctions/AggregateFunctionIntervalLengthSum.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -7,6 +9,7 @@ #include #include #include +#include #include #include @@ -14,8 +17,6 @@ #include #include -#include - namespace DB { @@ -67,7 +68,7 @@ struct AggregateFunctionIntervalLengthSumData /// either sort whole container or do so partially merging ranges afterwards if (!sorted && !other.sorted) { - std::sort(std::begin(segments), std::end(segments)); + ::sort(std::begin(segments), std::end(segments)); } else { @@ -76,10 +77,10 @@ struct AggregateFunctionIntervalLengthSumData const auto end = std::end(segments); if (!sorted) - std::sort(begin, middle); + ::sort(begin, middle); if (!other.sorted) - std::sort(middle, end); + ::sort(middle, end); std::inplace_merge(begin, middle, end); } @@ -89,11 +90,11 @@ struct AggregateFunctionIntervalLengthSumData void sort() { - if (!sorted) - { - std::sort(std::begin(segments), std::end(segments)); - sorted = true; - } + if (sorted) + return; + + ::sort(std::begin(segments), std::end(segments)); + sorted = true; } void serialize(WriteBuffer & buf) const diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index 550fd138452..79c18944620 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -226,7 +227,7 @@ public: { keys.push_back(it.first); } - std::sort(keys.begin(), keys.end()); + ::sort(keys.begin(), keys.end()); // insert using sorted keys to result column for (auto & key : keys) diff --git a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h index 8755c9db5db..18af4f0a220 100644 --- a/src/AggregateFunctions/AggregateFunctionMaxIntersections.h +++ b/src/AggregateFunctions/AggregateFunctionMaxIntersections.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -142,7 +143,7 @@ public: auto & array = this->data(place).value; /// Sort by position; for equal position, sort by weight to get deterministic result. - std::sort(array.begin(), array.end()); + ::sort(array.begin(), array.end()); for (const auto & point_weight : array) { diff --git a/src/AggregateFunctions/AggregateFunctionSequenceMatch.h b/src/AggregateFunctions/AggregateFunctionSequenceMatch.h index 04634839751..248454bef02 100644 --- a/src/AggregateFunctions/AggregateFunctionSequenceMatch.h +++ b/src/AggregateFunctions/AggregateFunctionSequenceMatch.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -74,11 +75,11 @@ struct AggregateFunctionSequenceMatchData final void sort() { - if (!sorted) - { - std::sort(std::begin(events_list), std::end(events_list), Comparator{}); - sorted = true; - } + if (sorted) + return; + + ::sort(std::begin(events_list), std::end(events_list), Comparator{}); + sorted = true; } void serialize(WriteBuffer & buf) const diff --git a/src/AggregateFunctions/AggregateFunctionSimpleState.h b/src/AggregateFunctions/AggregateFunctionSimpleState.h index d32d9a4f806..d63d8b71b8c 100644 --- a/src/AggregateFunctions/AggregateFunctionSimpleState.h +++ b/src/AggregateFunctions/AggregateFunctionSimpleState.h @@ -17,15 +17,11 @@ class AggregateFunctionSimpleState final : public IAggregateFunctionHelper(arguments_, params_) , nested_func(nested_) - , arguments(arguments_) - , params(params_) { } @@ -35,18 +31,19 @@ public: { DataTypeCustomSimpleAggregateFunction::checkSupportedFunctions(nested_func); - // Need to make a clone because it'll be customized. - auto storage_type = DataTypeFactory::instance().get(nested_func->getReturnType()->getName()); - + // Need to make a clone to avoid recursive reference. + auto storage_type_out = DataTypeFactory::instance().get(nested_func->getReturnType()->getName()); // Need to make a new function with promoted argument types because SimpleAggregates requires arg_type = return_type. AggregateFunctionProperties properties; auto function - = AggregateFunctionFactory::instance().get(nested_func->getName(), {storage_type}, nested_func->getParameters(), properties); + = AggregateFunctionFactory::instance().get(nested_func->getName(), {storage_type_out}, nested_func->getParameters(), properties); + // Need to make a clone because it'll be customized. + auto storage_type_arg = DataTypeFactory::instance().get(nested_func->getReturnType()->getName()); DataTypeCustomNamePtr custom_name - = std::make_unique(function, DataTypes{nested_func->getReturnType()}, params); - storage_type->setCustomization(std::make_unique(std::move(custom_name), nullptr)); - return storage_type; + = std::make_unique(function, DataTypes{nested_func->getReturnType()}, parameters); + storage_type_arg->setCustomization(std::make_unique(std::move(custom_name), nullptr)); + return storage_type_arg; } bool isVersioned() const override diff --git a/src/AggregateFunctions/AggregateFunctionState.h b/src/AggregateFunctions/AggregateFunctionState.h index 98fcfa83d67..f4f55835c93 100644 --- a/src/AggregateFunctions/AggregateFunctionState.h +++ b/src/AggregateFunctions/AggregateFunctionState.h @@ -20,13 +20,12 @@ class AggregateFunctionState final : public IAggregateFunctionHelper(arguments_, params_) - , nested_func(nested_), arguments(arguments_), params(params_) {} + , nested_func(nested_) + {} String getName() const override { diff --git a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp index c6e7029d283..83a91ef06fc 100644 --- a/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionStudentTTest.cpp @@ -7,6 +7,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -27,14 +28,24 @@ struct StudentTTestData : public TTestMoments { static constexpr auto name = "studentTTest"; - std::pair getResult() const + bool hasEnoughObservations() const { - Float64 mean_x = x1 / nx; - Float64 mean_y = y1 / ny; + return nx > 0 && ny > 0 && nx + ny > 2; + } + + Float64 getDegreesOfFreedom() const + { + return nx + ny - 2; + } + + std::tuple getResult() const + { + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); /// To estimate the variance we first estimate two means. /// That's why the number of degrees of freedom is the total number of values of both samples minus 2. - Float64 degrees_of_freedom = nx + ny - 2; + Float64 degrees_of_freedom = getDegreesOfFreedom(); /// Calculate s^2 /// The original formulae looks like @@ -59,12 +70,14 @@ AggregateFunctionPtr createAggregateFunctionStudentTTest( const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { assertBinary(name, argument_types); - assertNoParameters(name, parameters); + + if (parameters.size() > 1) + throw Exception("Aggregate function " + name + " requires zero or one parameter.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!isNumber(argument_types[0]) || !isNumber(argument_types[1])) throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::BAD_ARGUMENTS); - return std::make_shared>(argument_types); + return std::make_shared>(argument_types, parameters); } } diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index 7e661a92c5b..295258cd8cf 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -226,7 +226,7 @@ public: { // FIXME why is storing NearestFieldType not enough, and we // have to check for decimals again here? - UInt32 scale = static_cast &>(key_column).getData().getScale(); + UInt32 scale = static_cast &>(key_column).getScale(); it = merged_maps.find(DecimalField(key, scale)); } else @@ -251,7 +251,7 @@ public: if constexpr (is_decimal) { - UInt32 scale = static_cast &>(key_column).getData().getScale(); + UInt32 scale = static_cast &>(key_column).getScale(); merged_maps.emplace(DecimalField(key, scale), std::move(new_values)); } else diff --git a/src/AggregateFunctions/AggregateFunctionTTest.h b/src/AggregateFunctions/AggregateFunctionTTest.h index a91ce16c3ff..40cf0023878 100644 --- a/src/AggregateFunctions/AggregateFunctionTTest.h +++ b/src/AggregateFunctions/AggregateFunctionTTest.h @@ -9,6 +9,7 @@ #include #include #include +#include /// This function is used in implementations of different T-Tests. @@ -28,6 +29,11 @@ struct Settings; class ReadBuffer; class WriteBuffer; +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + /** * If you have a cumulative distribution function F, then calculating the p-value for given statistic T is simply 1−F(T) * In our case p-value is two-sided, so we multiply it by 2. @@ -79,10 +85,29 @@ template class AggregateFunctionTTest : public IAggregateFunctionDataHelper> { +private: + bool need_confidence_interval = false; + Float64 confidence_level; public: - AggregateFunctionTTest(const DataTypes & arguments) - : IAggregateFunctionDataHelper>({arguments}, {}) + AggregateFunctionTTest(const DataTypes & arguments, const Array & params) + : IAggregateFunctionDataHelper>({arguments}, params) { + if (params.size() > 0) + { + need_confidence_interval = true; + confidence_level = params.at(0).safeGet(); + + if (!std::isfinite(confidence_level)) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function {} requires finite parameter values.", Data::name); + } + + if (confidence_level <= 0.0 || confidence_level >= 1.0 || fabs(confidence_level - 0.0) < DBL_EPSILON || fabs(confidence_level - 1.0) < DBL_EPSILON) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Confidence level parameter must be between 0 and 1 in aggregate function {}.", Data::name); + } + + } } String getName() const override @@ -92,22 +117,48 @@ public: DataTypePtr getReturnType() const override { - DataTypes types + if (need_confidence_interval) { - std::make_shared>(), - std::make_shared>(), - }; + DataTypes types + { + std::make_shared>(), + std::make_shared>(), + std::make_shared>(), + std::make_shared>(), + }; - Strings names + Strings names + { + "t_statistic", + "p_value", + "confidence_interval_low", + "confidence_interval_high", + }; + + return std::make_shared( + std::move(types), + std::move(names) + ); + } + else { - "t_statistic", - "p_value" - }; + DataTypes types + { + std::make_shared>(), + std::make_shared>(), + }; - return std::make_shared( - std::move(types), - std::move(names) - ); + Strings names + { + "t_statistic", + "p_value", + }; + + return std::make_shared( + std::move(types), + std::move(names) + ); + } } bool allocatesMemoryInArena() const override { return false; } @@ -140,17 +191,46 @@ public: void insertResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena *) const override { - auto [t_statistic, p_value] = this->data(place).getResult(); + auto & data = this->data(place); + auto & column_tuple = assert_cast(to); + + if (!data.hasEnoughObservations() || data.isEssentiallyConstant()) + { + auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); + auto & column_value = assert_cast &>(column_tuple.getColumn(1)); + column_stat.getData().push_back(std::numeric_limits::quiet_NaN()); + column_value.getData().push_back(std::numeric_limits::quiet_NaN()); + + if (need_confidence_interval) + { + auto & column_ci_low = assert_cast &>(column_tuple.getColumn(2)); + auto & column_ci_high = assert_cast &>(column_tuple.getColumn(3)); + column_ci_low.getData().push_back(std::numeric_limits::quiet_NaN()); + column_ci_high.getData().push_back(std::numeric_limits::quiet_NaN()); + } + + return; + } + + auto [t_statistic, p_value] = data.getResult(); /// Because p-value is a probability. p_value = std::min(1.0, std::max(0.0, p_value)); - auto & column_tuple = assert_cast(to); auto & column_stat = assert_cast &>(column_tuple.getColumn(0)); auto & column_value = assert_cast &>(column_tuple.getColumn(1)); column_stat.getData().push_back(t_statistic); column_value.getData().push_back(p_value); + + if (need_confidence_interval) + { + auto [ci_low, ci_high] = data.getConfidenceIntervals(confidence_level, data.getDegreesOfFreedom()); + auto & column_ci_low = assert_cast &>(column_tuple.getColumn(2)); + auto & column_ci_high = assert_cast &>(column_tuple.getColumn(3)); + column_ci_low.getData().push_back(ci_low); + column_ci_high.getData().push_back(ci_high); + } } }; diff --git a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp index c7c56da79f6..fe5cf83c509 100644 --- a/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp +++ b/src/AggregateFunctions/AggregateFunctionWelchTTest.cpp @@ -7,6 +7,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -21,34 +22,38 @@ struct WelchTTestData : public TTestMoments { static constexpr auto name = "welchTTest"; - std::pair getResult() const + bool hasEnoughObservations() const { - Float64 mean_x = x1 / nx; - Float64 mean_y = y1 / ny; + return nx > 1 && ny > 1; + } - /// s_x^2, s_y^2 - - /// The original formulae looks like \frac{1}{size_x - 1} \sum_{i = 1}^{size_x}{(x_i - \bar{x}) ^ 2} - /// But we made some mathematical transformations not to store original sequences. - /// Also we dropped sqrt, because later it will be squared later. + Float64 getDegreesOfFreedom() const + { + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); Float64 sx2 = (x2 + nx * mean_x * mean_x - 2 * mean_x * x1) / (nx - 1); Float64 sy2 = (y2 + ny * mean_y * mean_y - 2 * mean_y * y1) / (ny - 1); - /// t-statistic - Float64 t_stat = (mean_x - mean_y) / sqrt(sx2 / nx + sy2 / ny); - - /// degrees of freedom - Float64 numerator_sqrt = sx2 / nx + sy2 / ny; Float64 numerator = numerator_sqrt * numerator_sqrt; Float64 denominator_x = sx2 * sx2 / (nx * nx * (nx - 1)); Float64 denominator_y = sy2 * sy2 / (ny * ny * (ny - 1)); - Float64 degrees_of_freedom = numerator / (denominator_x + denominator_y); + return numerator / (denominator_x + denominator_y); + } - return {t_stat, getPValue(degrees_of_freedom, t_stat * t_stat)}; + std::tuple getResult() const + { + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); + + /// t-statistic + Float64 se = getStandardError(); + Float64 t_stat = (mean_x - mean_y) / se; + + return {t_stat, getPValue(getDegreesOfFreedom(), t_stat * t_stat)}; } }; @@ -56,12 +61,14 @@ AggregateFunctionPtr createAggregateFunctionWelchTTest( const std::string & name, const DataTypes & argument_types, const Array & parameters, const Settings *) { assertBinary(name, argument_types); - assertNoParameters(name, parameters); + + if (parameters.size() > 1) + throw Exception("Aggregate function " + name + " requires zero or one parameter.", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); if (!isNumber(argument_types[0]) || !isNumber(argument_types[1])) throw Exception("Aggregate function " + name + " only supports numerical types", ErrorCodes::BAD_ARGUMENTS); - return std::make_shared>(argument_types); + return std::make_shared>(argument_types, parameters); } } diff --git a/src/AggregateFunctions/Moments.h b/src/AggregateFunctions/Moments.h index d2a6b0b5581..45a77e9cfdb 100644 --- a/src/AggregateFunctions/Moments.h +++ b/src/AggregateFunctions/Moments.h @@ -2,7 +2,9 @@ #include #include +#include #include +#include namespace DB @@ -358,6 +360,50 @@ struct TTestMoments { readPODBinary(*this, buf); } + + Float64 getMeanX() const + { + return x1 / nx; + } + + Float64 getMeanY() const + { + return y1 / ny; + } + + Float64 getStandardError() const + { + /// The original formulae looks like \frac{1}{size_x - 1} \sum_{i = 1}^{size_x}{(x_i - \bar{x}) ^ 2} + /// But we made some mathematical transformations not to store original sequences. + /// Also we dropped sqrt, because later it will be squared later. + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); + + Float64 sx2 = (x2 + nx * mean_x * mean_x - 2 * mean_x * x1) / (nx - 1); + Float64 sy2 = (y2 + ny * mean_y * mean_y - 2 * mean_y * y1) / (ny - 1); + + return sqrt(sx2 / nx + sy2 / ny); + } + + std::pair getConfidenceIntervals(Float64 confidence_level, Float64 degrees_of_freedom) const + { + Float64 mean_x = getMeanX(); + Float64 mean_y = getMeanY(); + Float64 se = getStandardError(); + + boost::math::students_t dist(degrees_of_freedom); + Float64 t = boost::math::quantile(boost::math::complement(dist, (1.0 - confidence_level) / 2.0)); + Float64 mean_diff = mean_x - mean_y; + Float64 ci_low = mean_diff - t * se; + Float64 ci_high = mean_diff + t * se; + + return {ci_low, ci_high}; + } + + bool isEssentiallyConstant() const + { + return getStandardError() < 10 * DBL_EPSILON * std::max(std::abs(getMeanX()), std::abs(getMeanY())); + } }; template diff --git a/src/AggregateFunctions/QuantileBFloat16Histogram.h b/src/AggregateFunctions/QuantileBFloat16Histogram.h index 2a71522c1fc..e60945f32ad 100644 --- a/src/AggregateFunctions/QuantileBFloat16Histogram.h +++ b/src/AggregateFunctions/QuantileBFloat16Histogram.h @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -134,7 +135,7 @@ private: ++arr_it; } - std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); + ::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); Float64 threshold = std::ceil(sum_weight * level); Float64 accumulated = 0; @@ -175,7 +176,7 @@ private: ++arr_it; } - std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); + ::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); size_t level_index = 0; Float64 accumulated = 0; diff --git a/src/AggregateFunctions/QuantileExact.h b/src/AggregateFunctions/QuantileExact.h index 4dcac2472b9..9be24689d12 100644 --- a/src/AggregateFunctions/QuantileExact.h +++ b/src/AggregateFunctions/QuantileExact.h @@ -88,7 +88,7 @@ struct QuantileExact : QuantileExactBase> if (!array.empty()) { size_t n = level < 1 ? level * array.size() : (array.size() - 1); - nth_element(array.begin(), array.begin() + n, array.end()); /// NOTE: You can think of the radix-select algorithm. + ::nth_element(array.begin(), array.begin() + n, array.end()); /// NOTE: You can think of the radix-select algorithm. return array[n]; } @@ -107,7 +107,7 @@ struct QuantileExact : QuantileExactBase> auto level = levels[indices[i]]; size_t n = level < 1 ? level * array.size() : (array.size() - 1); - nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + ::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); result[indices[i]] = array[n]; prev_n = n; } @@ -143,7 +143,7 @@ struct QuantileExactExclusive : public QuantileExact else if (n < 1) return static_cast(array[0]); - nth_element(array.begin(), array.begin() + n - 1, array.end()); + ::nth_element(array.begin(), array.begin() + n - 1, array.end()); auto nth_elem = std::min_element(array.begin() + n, array.end()); return static_cast(array[n - 1]) + (h - n) * static_cast(*nth_elem - array[n - 1]); @@ -172,7 +172,7 @@ struct QuantileExactExclusive : public QuantileExact result[indices[i]] = static_cast(array[0]); else { - nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); + ::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); auto nth_elem = std::min_element(array.begin() + n, array.end()); result[indices[i]] = static_cast(array[n - 1]) + (h - n) * static_cast(*nth_elem - array[n - 1]); @@ -207,7 +207,7 @@ struct QuantileExactInclusive : public QuantileExact return static_cast(array[array.size() - 1]); else if (n < 1) return static_cast(array[0]); - nth_element(array.begin(), array.begin() + n - 1, array.end()); + ::nth_element(array.begin(), array.begin() + n - 1, array.end()); auto nth_elem = std::min_element(array.begin() + n, array.end()); return static_cast(array[n - 1]) + (h - n) * static_cast(*nth_elem - array[n - 1]); @@ -234,7 +234,7 @@ struct QuantileExactInclusive : public QuantileExact result[indices[i]] = static_cast(array[0]); else { - nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); + ::nth_element(array.begin() + prev_n, array.begin() + n - 1, array.end()); auto nth_elem = std::min_element(array.begin() + n, array.end()); result[indices[i]] = static_cast(array[n - 1]) + (h - n) * (static_cast(*nth_elem) - array[n - 1]); @@ -262,9 +262,7 @@ struct QuantileExactLow : public QuantileExactBase(floor(s / 2))]; + n = static_cast(floor(s / 2)); } else { - return array[static_cast((floor(s / 2)) - 1)]; + n = static_cast((floor(s / 2)) - 1); } } else @@ -284,9 +282,10 @@ struct QuantileExactLow : public QuantileExactBase::quiet_NaN(); } @@ -295,12 +294,11 @@ struct QuantileExactLow : public QuantileExactBase(floor(s / 2))]; + n = static_cast(floor(s / 2)); } else { - result[indices[i]] = array[static_cast(floor((s / 2) - 1))]; + n = static_cast(floor((s / 2) - 1)); } } else { // else quantile is the nth index of the sorted array obtained by multiplying // level and size of array. Example if level = 0.1 and size of array is 10. - size_t n = level < 1 ? level * array.size() : (array.size() - 1); - result[indices[i]] = array[n]; + n = level < 1 ? level * array.size() : (array.size() - 1); } + ::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + result[indices[i]] = array[n]; + prev_n = n; } } else @@ -344,23 +344,22 @@ struct QuantileExactHigh : public QuantileExactBase(floor(s / 2))]; + n = static_cast(floor(s / 2)); } else { // else quantile is the nth index of the sorted array obtained by multiplying // level and size of array. Example if level = 0.1 and size of array is 10. - size_t n = level < 1 ? level * array.size() : (array.size() - 1); - return array[n]; + n = level < 1 ? level * array.size() : (array.size() - 1); } + ::nth_element(array.begin(), array.begin() + n, array.end()); + return array[n]; } return std::numeric_limits::quiet_NaN(); } @@ -369,26 +368,27 @@ struct QuantileExactHigh : public QuantileExactBase(floor(s / 2))]; + n = static_cast(floor(s / 2)); } else { // else quantile is the nth index of the sorted array obtained by multiplying // level and size of array. Example if level = 0.1 and size of array is 10. - size_t n = level < 1 ? level * array.size() : (array.size() - 1); - result[indices[i]] = array[n]; + n = level < 1 ? level * array.size() : (array.size() - 1); } + ::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + result[indices[i]] = array[n]; + prev_n = n; } } else diff --git a/src/AggregateFunctions/QuantileExactWeighted.h b/src/AggregateFunctions/QuantileExactWeighted.h index 97ad15f7e75..69e716f09ae 100644 --- a/src/AggregateFunctions/QuantileExactWeighted.h +++ b/src/AggregateFunctions/QuantileExactWeighted.h @@ -1,5 +1,7 @@ #pragma once +#include + #include #include @@ -101,7 +103,7 @@ struct QuantileExactWeighted ++i; } - std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); + ::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); Float64 threshold = std::ceil(sum_weight * level); Float64 accumulated = 0; @@ -151,7 +153,7 @@ struct QuantileExactWeighted ++i; } - std::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); + ::sort(array, array + size, [](const Pair & a, const Pair & b) { return a.first < b.first; }); Float64 accumulated = 0; diff --git a/src/AggregateFunctions/QuantileTiming.h b/src/AggregateFunctions/QuantileTiming.h index 36f1da3ee60..c89d1b66f5b 100644 --- a/src/AggregateFunctions/QuantileTiming.h +++ b/src/AggregateFunctions/QuantileTiming.h @@ -90,7 +90,7 @@ namespace detail /** This function must be called before get-functions. */ void prepare() const { - std::sort(elems, elems + count); + ::sort(elems, elems + count); } UInt16 get(double level) const @@ -183,7 +183,7 @@ namespace detail /// Sorting an array will not be considered a violation of constancy. auto & array = elems; - nth_element(array.begin(), array.begin() + n, array.end()); + ::nth_element(array.begin(), array.begin() + n, array.end()); quantile = array[n]; } @@ -204,7 +204,7 @@ namespace detail ? level * elems.size() : (elems.size() - 1); - nth_element(array.begin() + prev_n, array.begin() + n, array.end()); + ::nth_element(array.begin() + prev_n, array.begin() + n, array.end()); result[level_index] = array[n]; prev_n = n; diff --git a/src/AggregateFunctions/QuantilesCommon.h b/src/AggregateFunctions/QuantilesCommon.h index 8a1645c3781..161e25d0dc2 100644 --- a/src/AggregateFunctions/QuantilesCommon.h +++ b/src/AggregateFunctions/QuantilesCommon.h @@ -2,6 +2,8 @@ #include +#include + #include #include @@ -64,7 +66,7 @@ struct QuantileLevels permutation[i] = i; } - std::sort(permutation.begin(), permutation.end(), [this] (size_t a, size_t b) { return levels[a] < levels[b]; }); + ::sort(permutation.begin(), permutation.end(), [this] (size_t a, size_t b) { return levels[a] < levels[b]; }); } }; diff --git a/src/AggregateFunctions/ReservoirSampler.h b/src/AggregateFunctions/ReservoirSampler.h index d113d659520..5f7ac13d908 100644 --- a/src/AggregateFunctions/ReservoirSampler.h +++ b/src/AggregateFunctions/ReservoirSampler.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -15,6 +16,7 @@ #include #include + namespace DB { struct Settings; @@ -237,6 +239,7 @@ private: UInt64 genRandom(size_t lim) { + assert(lim > 0); /// With a large number of values, we will generate random numbers several times slower. if (lim <= static_cast(rng.max())) return static_cast(rng()) % static_cast(lim); @@ -249,7 +252,7 @@ private: if (sorted) return; sorted = true; - std::sort(samples.begin(), samples.end(), Comparer()); + ::sort(samples.begin(), samples.end(), Comparer()); } template diff --git a/src/AggregateFunctions/ReservoirSamplerDeterministic.h b/src/AggregateFunctions/ReservoirSamplerDeterministic.h index ca716b24ce2..2baeea76996 100644 --- a/src/AggregateFunctions/ReservoirSamplerDeterministic.h +++ b/src/AggregateFunctions/ReservoirSamplerDeterministic.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -258,7 +259,9 @@ private: { if (sorted) return; - std::sort(samples.begin(), samples.end(), [](const auto & lhs, const auto & rhs) { return lhs.first < rhs.first; }); + + /// In order to provide deterministic result we must sort by value and hash + ::sort(samples.begin(), samples.end(), [](const auto & lhs, const auto & rhs) { return lhs < rhs; }); sorted = true; } diff --git a/src/AggregateFunctions/StatCommon.h b/src/AggregateFunctions/StatCommon.h index a8fc7c53072..d670e646f4b 100644 --- a/src/AggregateFunctions/StatCommon.h +++ b/src/AggregateFunctions/StatCommon.h @@ -1,13 +1,17 @@ #pragma once -#include -#include -#include - #include #include #include +#include + +#include + +#include +#include + + namespace DB { struct Settings; @@ -41,7 +45,7 @@ std::pair computeRanksAndTieCorrection(const Values & value /// Save initial positions, than sort indices according to the values. std::vector indexes(size); std::iota(indexes.begin(), indexes.end(), 0); - std::sort(indexes.begin(), indexes.end(), + ::sort(indexes.begin(), indexes.end(), [&] (size_t lhs, size_t rhs) { return values[lhs] < values[rhs]; }); size_t left = 0; diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index 5da87cfd6f7..c26eec440e6 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -15,6 +15,7 @@ #include #include #include +#include #include #include @@ -632,7 +633,7 @@ BackupEntries makeBackupEntries(const Elements & elements, const ContextPtr & co throw Exception("Backup must not be empty", ErrorCodes::BACKUP_IS_EMPTY); /// Check that all backup entries are unique. - std::sort( + ::sort( backup_entries.begin(), backup_entries.end(), [](const std::pair> & lhs, const std::pair> & rhs) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 57d4bf29491..b99ffd7ee18 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -79,6 +79,7 @@ set(dbms_sources) add_headers_and_sources(clickhouse_common_io Common) add_headers_and_sources(clickhouse_common_io Common/HashTable) add_headers_and_sources(clickhouse_common_io IO) +add_headers_and_sources(clickhouse_common_io IO/Archives) add_headers_and_sources(clickhouse_common_io IO/S3) list (REMOVE_ITEM clickhouse_common_io_sources Common/malloc.cpp Common/new_delete.cpp) @@ -475,11 +476,6 @@ if (TARGET ch_contrib::sqlite) dbms_target_link_libraries(PUBLIC ch_contrib::sqlite) endif() -if (USE_CASSANDRA) - dbms_target_link_libraries(PUBLIC ${CASSANDRA_LIBRARY}) - dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR}) -endif() - if (TARGET ch_contrib::msgpack) target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::msgpack) endif() @@ -513,6 +509,10 @@ if (TARGET ch_contrib::bzip2) target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::bzip2) endif() +if (TARGET ch_contrib::minizip) + target_link_libraries (clickhouse_common_io PRIVATE ch_contrib::minizip) +endif () + if (TARGET ch_contrib::simdjson) dbms_target_link_libraries(PRIVATE ch_contrib::simdjson) endif() diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 0a30e924ffb..6fd8c0cb9bc 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include #include @@ -552,6 +553,37 @@ void ClientBase::initLogsOutputStream() } } +void ClientBase::updateSuggest(const ASTCreateQuery & ast_create) +{ + std::vector new_words; + + if (ast_create.database) + new_words.push_back(ast_create.getDatabase()); + new_words.push_back(ast_create.getTable()); + + if (ast_create.columns_list && ast_create.columns_list->columns) + { + for (const auto & elem : ast_create.columns_list->columns->children) + { + if (const auto * column = elem->as()) + new_words.push_back(column->name); + } + } + + suggest->addWords(std::move(new_words)); +} + +bool ClientBase::isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context) +{ + if (!insert_query.data) + return false; + + auto settings = context->getSettings(); + if (insert_query.settings_ast) + settings.applyChanges(insert_query.settings_ast->as()->changes); + + return !settings.async_insert; +} void ClientBase::processTextAsSingleQuery(const String & full_query) { @@ -565,10 +597,24 @@ void ClientBase::processTextAsSingleQuery(const String & full_query) String query_to_execute; - // An INSERT query may have the data that follow query text. Remove the - /// Send part of query without data, because data will be sent separately. - auto * insert = parsed_query->as(); - if (insert && insert->data) + /// Query will be parsed before checking the result because error does not + /// always means a problem, i.e. if table already exists, and it is no a + /// huge problem if suggestion will be added even on error, since this is + /// just suggestion. + if (auto * create = parsed_query->as()) + { + /// Do not update suggest, until suggestion will be ready + /// (this will avoid extra complexity) + if (suggest) + updateSuggest(*create); + } + + /// An INSERT query may have the data that follows query text. + /// Send part of the query without data, because data will be sent separately. + /// But for asynchronous inserts we don't extract data, because it's needed + /// to be done on server side in that case (for coalescing the data from multiple inserts on server side). + const auto * insert = parsed_query->as(); + if (insert && isSyncInsertWithData(*insert, global_context)) query_to_execute = full_query.substr(0, insert->data - full_query.data()); else query_to_execute = full_query; @@ -1229,7 +1275,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin for (const auto & query_id_format : query_id_formats) { writeString(query_id_format.first, std_out); - writeString(fmt::format(query_id_format.second, fmt::arg("query_id", global_context->getCurrentQueryId())), std_out); + writeString(fmt::format(fmt::runtime(query_id_format.second), fmt::arg("query_id", global_context->getCurrentQueryId())), std_out); writeChar('\n', std_out); std_out.next(); } @@ -1271,8 +1317,10 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin if (insert && insert->select) insert->tryFindInputFunction(input_function); + bool is_async_insert = global_context->getSettingsRef().async_insert && insert && insert->hasInlinedData(); + /// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately. - if (insert && (!insert->select || input_function) && !insert->watch) + if (insert && (!insert->select || input_function) && !insert->watch && !is_async_insert) { if (input_function && insert->format.empty()) throw Exception("FORMAT must be specified for function input()", ErrorCodes::INVALID_USAGE_OF_INPUT); @@ -1402,17 +1450,17 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText( // row input formats (e.g. TSV) can't tell when the input stops, // unlike VALUES. auto * insert_ast = parsed_query->as(); + const char * query_to_execute_end = this_query_end; + if (insert_ast && insert_ast->data) { this_query_end = find_first_symbols<'\n'>(insert_ast->data, all_queries_end); insert_ast->end = this_query_end; - query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), insert_ast->data - this_query_begin); - } - else - { - query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), this_query_end - this_query_begin); + query_to_execute_end = isSyncInsertWithData(*insert_ast, global_context) ? insert_ast->data : this_query_end; } + query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), query_to_execute_end - this_query_begin); + // Try to include the trailing comment with test hints. It is just // a guess for now, because we don't yet know where the query ends // if it is an INSERT query with inline data. We will do it again @@ -1453,6 +1501,25 @@ String ClientBase::prompt() const } +void ClientBase::initQueryIdFormats() +{ + if (!query_id_formats.empty()) + return; + + /// Initialize query_id_formats if any + if (config().has("query_id_formats")) + { + Poco::Util::AbstractConfiguration::Keys keys; + config().keys("query_id_formats", keys); + for (const auto & name : keys) + query_id_formats.emplace_back(name + ":", config().getString("query_id_formats." + name)); + } + + if (query_id_formats.empty()) + query_id_formats.emplace_back("Query id:", " {query_id}\n"); +} + + void ClientBase::runInteractive() { if (config().has("query_id")) @@ -1460,10 +1527,11 @@ void ClientBase::runInteractive() if (print_time_to_stderr) throw Exception("time option could be specified only in non-interactive mode", ErrorCodes::BAD_ARGUMENTS); + initQueryIdFormats(); + /// Initialize DateLUT here to avoid counting time spent here as query execution time. const auto local_tz = DateLUT::instance().getTimeZone(); - std::optional suggest; suggest.emplace(); if (load_suggestions) { @@ -1481,18 +1549,6 @@ void ClientBase::runInteractive() home_path = home_path_cstr; } - /// Initialize query_id_formats if any - if (config().has("query_id_formats")) - { - Poco::Util::AbstractConfiguration::Keys keys; - config().keys("query_id_formats", keys); - for (const auto & name : keys) - query_id_formats.emplace_back(name + ":", config().getString("query_id_formats." + name)); - } - - if (query_id_formats.empty()) - query_id_formats.emplace_back("Query id:", " {query_id}\n"); - /// Load command history if present. if (config().has("history_file")) history_file = config().getString("history_file"); @@ -1601,6 +1657,9 @@ void ClientBase::runInteractive() void ClientBase::runNonInteractive() { + if (delayed_interactive) + initQueryIdFormats(); + if (!queries_files.empty()) { auto process_multi_query_from_file = [&](const String & file) @@ -1662,7 +1721,12 @@ void ClientBase::showClientVersion() } -void ClientBase::readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector & external_tables_arguments) +void ClientBase::readArguments( + int argc, + char ** argv, + Arguments & common_arguments, + std::vector & external_tables_arguments, + std::vector & hosts_and_ports_arguments) { /** We allow different groups of arguments: * - common arguments; @@ -1673,6 +1737,10 @@ void ClientBase::readArguments(int argc, char ** argv, Arguments & common_argume */ bool in_external_group = false; + + std::string prev_host_arg; + std::string prev_port_arg; + for (int arg_num = 1; arg_num < argc; ++arg_num) { const char * arg = argv[arg_num]; @@ -1733,10 +1801,74 @@ void ClientBase::readArguments(int argc, char ** argv, Arguments & common_argume query_parameters.emplace(String(param_continuation), String(arg)); } } + else if (startsWith(arg, "--host") || startsWith(arg, "-h")) + { + std::string host_arg; + /// --host host + if (arg == "--host"sv || arg == "-h"sv) + { + ++arg_num; + if (arg_num >= argc) + throw Exception("Host argument requires value", ErrorCodes::BAD_ARGUMENTS); + arg = argv[arg_num]; + host_arg = "--host="; + host_arg.append(arg); + } + else + host_arg = arg; + + /// --port port1 --host host1 + if (!prev_port_arg.empty()) + { + hosts_and_ports_arguments.push_back({host_arg, prev_port_arg}); + prev_port_arg.clear(); + } + else + { + /// --host host1 --host host2 + if (!prev_host_arg.empty()) + hosts_and_ports_arguments.push_back({prev_host_arg}); + + prev_host_arg = host_arg; + } + } + else if (startsWith(arg, "--port")) + { + std::string port_arg = arg; + /// --port port + if (arg == "--port"sv) + { + port_arg.push_back('='); + ++arg_num; + if (arg_num >= argc) + throw Exception("Port argument requires value", ErrorCodes::BAD_ARGUMENTS); + arg = argv[arg_num]; + port_arg.append(arg); + } + + /// --host host1 --port port1 + if (!prev_host_arg.empty()) + { + hosts_and_ports_arguments.push_back({port_arg, prev_host_arg}); + prev_host_arg.clear(); + } + else + { + /// --port port1 --port port2 + if (!prev_port_arg.empty()) + hosts_and_ports_arguments.push_back({prev_port_arg}); + + prev_port_arg = port_arg; + } + } else common_arguments.emplace_back(arg); } } + if (!prev_host_arg.empty()) + hosts_and_ports_arguments.push_back({prev_host_arg}); + if (!prev_port_arg.empty()) + hosts_and_ports_arguments.push_back({prev_port_arg}); } void ClientBase::parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments) @@ -1779,8 +1911,9 @@ void ClientBase::init(int argc, char ** argv) Arguments common_arguments{""}; /// 0th argument is ignored. std::vector external_tables_arguments; + std::vector hosts_and_ports_arguments; - readArguments(argc, argv, common_arguments, external_tables_arguments); + readArguments(argc, argv, common_arguments, external_tables_arguments, hosts_and_ports_arguments); po::variables_map options; OptionsDescription options_description; @@ -1933,7 +2066,7 @@ void ClientBase::init(int argc, char ** argv) profile_events.print = options.count("print-profile-events"); profile_events.delay_ms = options["profile-events-delay-ms"].as(); - processOptions(options_description, options, external_tables_arguments); + processOptions(options_description, options, external_tables_arguments, hosts_and_ports_arguments); argsToConfig(common_arguments, config(), 100); clearPasswordFromCommandLine(argc, argv); diff --git a/src/Client/ClientBase.h b/src/Client/ClientBase.h index 1926df5afea..0a11745b996 100644 --- a/src/Client/ClientBase.h +++ b/src/Client/ClientBase.h @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -91,13 +92,15 @@ protected: { std::optional main_description; std::optional external_description; + std::optional hosts_and_ports_description; }; virtual void printHelpMessage(const OptionsDescription & options_description) = 0; virtual void addOptions(OptionsDescription & options_description) = 0; virtual void processOptions(const OptionsDescription & options_description, const CommandLineOptions & options, - const std::vector & external_tables_arguments) = 0; + const std::vector & external_tables_arguments, + const std::vector & hosts_and_ports_arguments) = 0; virtual void processConfig() = 0; protected: @@ -133,10 +136,21 @@ private: void resetOutput(); void outputQueryInfo(bool echo_query_); - void readArguments(int argc, char ** argv, Arguments & common_arguments, std::vector & external_tables_arguments); + void readArguments( + int argc, + char ** argv, + Arguments & common_arguments, + std::vector & external_tables_arguments, + std::vector & hosts_and_ports_arguments); void parseAndCheckOptions(OptionsDescription & options_description, po::variables_map & options, Arguments & arguments); + void updateSuggest(const ASTCreateQuery & ast_create); + + void initQueryIdFormats(); + protected: + static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context); + bool is_interactive = false; /// Use either interactive line editing interface or batch mode. bool is_multiquery = false; bool delayed_interactive = false; @@ -144,6 +158,8 @@ protected: bool echo_queries = false; /// Print queries before execution in batch mode. bool ignore_error = false; /// In case of errors, don't print error message, continue to next query. Only applicable for non-interactive mode. bool print_time_to_stderr = false; /// Output execution time to stderr in batch mode. + + std::optional suggest; bool load_suggestions = false; std::vector queries_files; /// If not empty, queries will be read from these files @@ -235,6 +251,14 @@ protected: } profile_events; QueryProcessingStage::Enum query_processing_stage; + + struct HostAndPort + { + String host; + UInt16 port; + }; + + std::vector hosts_and_ports{}; }; } diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 3a5d4f4cf33..5ad34ba8e81 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -6,7 +6,6 @@ #include #include - namespace DB { @@ -114,6 +113,7 @@ void highlight(const String & query, std::vector & colors {TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)}, + {TokenType::VerticalDelimiter, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)}, {TokenType::HereDoc, Replxx::Color::CYAN}, @@ -151,6 +151,11 @@ void highlight(const String & query, std::vector & colors for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken()) { + if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter) + ReplxxLineReader::setLastIsDelimiter(true); + else if (token.type != TokenType::Whitespace) + ReplxxLineReader::setLastIsDelimiter(false); + size_t utf8_len = UTF8::countCodePoints(reinterpret_cast(token.begin), token.size()); for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index) { diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 505a6514812..ad2fc76f090 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -405,7 +405,7 @@ bool Connection::ping() } catch (const Poco::Exception & e) { - LOG_TRACE(log_wrapper.get(), e.displayText()); + LOG_TRACE(log_wrapper.get(), fmt::runtime(e.displayText())); return false; } diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 4d27c9efc69..3385834e386 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -58,9 +58,9 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: auto table_status_it = status_response.table_states_by_id.find(*table_to_check); if (table_status_it == status_response.table_states_by_id.end()) { - const char * message_pattern = "There is no table {}.{} on server: {}"; - fail_message = fmt::format(message_pattern, backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription()); - LOG_WARNING(log, fail_message); + fail_message = fmt::format("There is no table {}.{} on server: {}", + backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription()); + LOG_WARNING(log, fmt::runtime(fail_message)); ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable); return; } diff --git a/src/Client/ConnectionParameters.cpp b/src/Client/ConnectionParameters.cpp index dbd463583f5..55569f080f6 100644 --- a/src/Client/ConnectionParameters.cpp +++ b/src/Client/ConnectionParameters.cpp @@ -23,15 +23,13 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfiguration & config) +ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfiguration & config, + std::string connection_host, + int connection_port) : host(connection_host), port(connection_port) { bool is_secure = config.getBool("secure", false); security = is_secure ? Protocol::Secure::Enable : Protocol::Secure::Disable; - host = config.getString("host", "localhost"); - port = config.getInt( - "port", config.getInt(is_secure ? "tcp_port_secure" : "tcp_port", is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT)); - default_database = config.getString("database", ""); /// changed the default value to "default" to fix the issue when the user in the prompt is blank @@ -61,12 +59,25 @@ ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfigurati /// By default compression is disabled if address looks like localhost. compression = config.getBool("compression", !isLocalAddress(DNSResolver::instance().resolveHost(host))) - ? Protocol::Compression::Enable : Protocol::Compression::Disable; + ? Protocol::Compression::Enable : Protocol::Compression::Disable; timeouts = ConnectionTimeouts( - Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0), - Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0), - Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0), - Poco::Timespan(config.getInt("tcp_keep_alive_timeout", 0), 0)); + Poco::Timespan(config.getInt("connect_timeout", DBMS_DEFAULT_CONNECT_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("send_timeout", DBMS_DEFAULT_SEND_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("receive_timeout", DBMS_DEFAULT_RECEIVE_TIMEOUT_SEC), 0), + Poco::Timespan(config.getInt("tcp_keep_alive_timeout", 0), 0)); +} + +ConnectionParameters::ConnectionParameters(const Poco::Util::AbstractConfiguration & config) + : ConnectionParameters(config, config.getString("host", "localhost"), getPortFromConfig(config)) +{ +} + +int ConnectionParameters::getPortFromConfig(const Poco::Util::AbstractConfiguration & config) +{ + bool is_secure = config.getBool("secure", false); + return config.getInt("port", + config.getInt(is_secure ? "tcp_port_secure" : "tcp_port", + is_secure ? DBMS_DEFAULT_SECURE_PORT : DBMS_DEFAULT_PORT)); } } diff --git a/src/Client/ConnectionParameters.h b/src/Client/ConnectionParameters.h index a169df8390a..dc509049c83 100644 --- a/src/Client/ConnectionParameters.h +++ b/src/Client/ConnectionParameters.h @@ -24,6 +24,9 @@ struct ConnectionParameters ConnectionParameters() {} ConnectionParameters(const Poco::Util::AbstractConfiguration & config); + ConnectionParameters(const Poco::Util::AbstractConfiguration & config, std::string host, int port); + + static int getPortFromConfig(const Poco::Util::AbstractConfiguration & config); }; } diff --git a/src/Client/LocalConnection.cpp b/src/Client/LocalConnection.cpp index 528c38f9b76..8ee4b9e1c1f 100644 --- a/src/Client/LocalConnection.cpp +++ b/src/Client/LocalConnection.cpp @@ -74,6 +74,8 @@ void LocalConnection::sendQuery( query_context->setProgressCallback([this] (const Progress & value) { return this->updateProgress(value); }); query_context->setFileProgressCallback([this](const FileProgress & value) { this->updateProgress(Progress(value)); }); } + if (!current_database.empty()) + query_context->setCurrentDatabase(current_database); CurrentThread::QueryScope query_scope_holder(query_context); @@ -427,9 +429,9 @@ void LocalConnection::getServerVersion( throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented"); } -void LocalConnection::setDefaultDatabase(const String &) +void LocalConnection::setDefaultDatabase(const String & database) { - throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Not implemented"); + current_database = database; } UInt64 LocalConnection::getServerRevision(const ConnectionTimeouts &) diff --git a/src/Client/LocalConnection.h b/src/Client/LocalConnection.h index 92c2af30c80..b85022cf183 100644 --- a/src/Client/LocalConnection.h +++ b/src/Client/LocalConnection.h @@ -142,5 +142,7 @@ private: /// Last "server" packet. std::optional next_packet_type; + + String current_database; }; } diff --git a/src/Client/MultiplexedConnections.cpp b/src/Client/MultiplexedConnections.cpp index 37a372dfb45..d1873ac038d 100644 --- a/src/Client/MultiplexedConnections.cpp +++ b/src/Client/MultiplexedConnections.cpp @@ -133,7 +133,7 @@ void MultiplexedConnections::sendQuery( modified_settings.group_by_two_level_threshold_bytes = 0; } - if (settings.allow_experimental_parallel_reading_from_replicas) + if (settings.max_parallel_replicas > 1 && settings.allow_experimental_parallel_reading_from_replicas) { client_info.collaborate_with_initiator = true; client_info.count_participating_replicas = replica_info.all_replicas_count; diff --git a/src/Client/Suggest.cpp b/src/Client/Suggest.cpp index b14af7ba8e9..738c98d2119 100644 --- a/src/Client/Suggest.cpp +++ b/src/Client/Suggest.cpp @@ -29,19 +29,21 @@ namespace ErrorCodes Suggest::Suggest() { /// Keywords may be not up to date with ClickHouse parser. - words = {"CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", - "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", - "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", - "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", - "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", - "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", - "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", - "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", - "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", - "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE", - "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", - "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", - "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE"}; + addWords({ + "CREATE", "DATABASE", "IF", "NOT", "EXISTS", "TEMPORARY", "TABLE", "ON", "CLUSTER", "DEFAULT", + "MATERIALIZED", "ALIAS", "ENGINE", "AS", "VIEW", "POPULATE", "SETTINGS", "ATTACH", "DETACH", "DROP", + "RENAME", "TO", "ALTER", "ADD", "MODIFY", "CLEAR", "COLUMN", "AFTER", "COPY", "PROJECT", + "PRIMARY", "KEY", "CHECK", "PARTITION", "PART", "FREEZE", "FETCH", "FROM", "SHOW", "INTO", + "OUTFILE", "FORMAT", "TABLES", "DATABASES", "LIKE", "PROCESSLIST", "CASE", "WHEN", "THEN", "ELSE", + "END", "DESCRIBE", "DESC", "USE", "SET", "OPTIMIZE", "FINAL", "DEDUPLICATE", "INSERT", "VALUES", + "SELECT", "DISTINCT", "SAMPLE", "ARRAY", "JOIN", "GLOBAL", "LOCAL", "ANY", "ALL", "INNER", + "LEFT", "RIGHT", "FULL", "OUTER", "CROSS", "USING", "PREWHERE", "WHERE", "GROUP", "BY", + "WITH", "TOTALS", "HAVING", "ORDER", "COLLATE", "LIMIT", "UNION", "AND", "OR", "ASC", + "IN", "KILL", "QUERY", "SYNC", "ASYNC", "TEST", "BETWEEN", "TRUNCATE", "USER", "ROLE", + "PROFILE", "QUOTA", "POLICY", "ROW", "GRANT", "REVOKE", "OPTION", "ADMIN", "EXCEPT", "REPLACE", + "IDENTIFIED", "HOST", "NAME", "READONLY", "WRITABLE", "PERMISSIVE", "FOR", "RESTRICTIVE", "RANDOMIZED", + "INTERVAL", "LIMITS", "ONLY", "TRACKING", "IP", "REGEXP", "ILIKE", + }); } static String getLoadSuggestionQuery(Int32 suggestion_limit, bool basic_suggestion) @@ -124,18 +126,6 @@ void Suggest::load(ContextPtr context, const ConnectionParameters & connection_p } /// Note that keyword suggestions are available even if we cannot load data from server. - - std::sort(words.begin(), words.end()); - words_no_case = words; - std::sort(words_no_case.begin(), words_no_case.end(), [](const std::string & str1, const std::string & str2) - { - return std::lexicographical_compare(begin(str1), end(str1), begin(str2), end(str2), [](const char char1, const char char2) - { - return std::tolower(char1) < std::tolower(char2); - }); - }); - - ready = true; }); } @@ -190,8 +180,14 @@ void Suggest::fillWordsFromBlock(const Block & block) const ColumnString & column = typeid_cast(*block.getByPosition(0).column); size_t rows = block.rows(); + + Words new_words; + new_words.reserve(rows); for (size_t i = 0; i < rows; ++i) - words.emplace_back(column.getDataAt(i).toString()); + { + new_words.emplace_back(column.getDataAt(i).toString()); + } + addWords(std::move(new_words)); } template diff --git a/src/Columns/Collator.cpp b/src/Columns/Collator.cpp index 312216054f5..0c2cdcf1baf 100644 --- a/src/Columns/Collator.cpp +++ b/src/Columns/Collator.cpp @@ -17,6 +17,7 @@ #include #include #include +#include namespace DB @@ -74,10 +75,10 @@ AvailableCollationLocales::LocalesVector AvailableCollationLocales::getAvailable result.push_back(name_and_locale.second); auto comparator = [] (const LocaleAndLanguage & f, const LocaleAndLanguage & s) - { - return f.locale_name < s.locale_name; - }; - std::sort(result.begin(), result.end(), comparator); + { + return f.locale_name < s.locale_name; + }; + ::sort(result.begin(), result.end(), comparator); return result; } diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 929c0153a0a..c4d75fed129 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -1,5 +1,3 @@ -#include // memcpy - #include #include #include @@ -9,12 +7,7 @@ #include #include #include - -#include -#include - #include - #include #include #include @@ -22,6 +15,9 @@ #include #include #include +#include +#include +#include // memcpy namespace DB @@ -54,12 +50,12 @@ ColumnArray::ColumnArray(MutableColumnPtr && nested_column, MutableColumnPtr && if (!offsets_concrete) throw Exception("offsets_column must be a ColumnUInt64", ErrorCodes::LOGICAL_ERROR); - if (!offsets_concrete->empty() && nested_column) + if (!offsets_concrete->empty() && data) { Offset last_offset = offsets_concrete->getData().back(); /// This will also prevent possible overflow in offset. - if (nested_column->size() != last_offset) + if (data->size() != last_offset) throw Exception("offsets_column has data inconsistent with nested_column", ErrorCodes::LOGICAL_ERROR); } @@ -127,18 +123,8 @@ size_t ColumnArray::size() const Field ColumnArray::operator[](size_t n) const { - size_t offset = offsetAt(n); - size_t size = sizeAt(n); - - if (size > max_array_size_as_field) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array of size {} is too large to be manipulated as single field, maximum size {}", - size, max_array_size_as_field); - - Array res(size); - - for (size_t i = 0; i < size; ++i) - res[i] = getData()[offset + i]; - + Field res; + get(n, res); return res; } @@ -152,11 +138,12 @@ void ColumnArray::get(size_t n, Field & res) const throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Array of size {} is too large to be manipulated as single field, maximum size {}", size, max_array_size_as_field); - res = Array(size); + res = Array(); Array & res_arr = DB::get(res); + res_arr.reserve(size); for (size_t i = 0; i < size; ++i) - getData().get(offset + i, res_arr[i]); + res_arr.push_back(getData()[offset + i]); } @@ -824,9 +811,9 @@ void ColumnArray::getPermutationImpl(size_t limit, Permutation & res, Comparator auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; }; if (limit) - partial_sort(res.begin(), res.begin() + limit, res.end(), less); + ::partial_sort(res.begin(), res.begin() + limit, res.end(), less); else - std::sort(res.begin(), res.end(), less); + ::sort(res.begin(), res.end(), less); } void ColumnArray::getPermutation(bool reverse, size_t limit, int nan_direction_hint, Permutation & res) const diff --git a/src/Columns/ColumnDecimal.cpp b/src/Columns/ColumnDecimal.cpp index 99085f0f976..9898f013886 100644 --- a/src/Columns/ColumnDecimal.cpp +++ b/src/Columns/ColumnDecimal.cpp @@ -9,7 +9,6 @@ #include #include - #include #include @@ -32,12 +31,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -template class DecimalPaddedPODArray; -template class DecimalPaddedPODArray; -template class DecimalPaddedPODArray; -template class DecimalPaddedPODArray; -template class DecimalPaddedPODArray; - template int ColumnDecimal::compareAt(size_t n, size_t m, const IColumn & rhs_, int) const { @@ -131,19 +124,6 @@ void ColumnDecimal::updateHashFast(SipHash & hash) const template void ColumnDecimal::getPermutation(bool reverse, size_t limit, int , IColumn::Permutation & res) const { -#if 1 /// TODO: perf test - if (data.size() <= std::numeric_limits::max()) - { - PaddedPODArray tmp_res; - permutation(reverse, limit, tmp_res); - - res.resize(tmp_res.size()); - for (size_t i = 0; i < tmp_res.size(); ++i) - res[i] = tmp_res[i]; - return; - } -#endif - permutation(reverse, limit, res); } @@ -151,7 +131,7 @@ template void ColumnDecimal::updatePermutation(bool reverse, size_t limit, int, IColumn::Permutation & res, EqualRanges & equal_ranges) const { auto equals = [this](size_t lhs, size_t rhs) { return data[lhs] == data[rhs]; }; - auto sort = [](auto begin, auto end, auto pred) { std::sort(begin, end, pred); }; + auto sort = [](auto begin, auto end, auto pred) { ::sort(begin, end, pred); }; auto partial_sort = [](auto begin, auto mid, auto end, auto pred) { ::partial_sort(begin, mid, end, pred); }; if (reverse) diff --git a/src/Columns/ColumnDecimal.h b/src/Columns/ColumnDecimal.h index b55083cd671..1712753bda2 100644 --- a/src/Columns/ColumnDecimal.h +++ b/src/Columns/ColumnDecimal.h @@ -1,66 +1,21 @@ #pragma once +#include + +#include +#include +#include +#include +#include +#include #include #include #include -#include -#include -#include -#include -#include -#include - -#include namespace DB { -/// PaddedPODArray extended by Decimal scale -template -class DecimalPaddedPODArray : public PaddedPODArray -{ -public: - using Base = PaddedPODArray; - using Base::operator[]; - - DecimalPaddedPODArray(size_t size, UInt32 scale_) - : Base(size), - scale(scale_) - {} - - DecimalPaddedPODArray(const DecimalPaddedPODArray & other) - : Base(other.begin(), other.end()), - scale(other.scale) - {} - - DecimalPaddedPODArray(DecimalPaddedPODArray && other) - { - this->swap(other); - std::swap(scale, other.scale); - } - - DecimalPaddedPODArray & operator=(DecimalPaddedPODArray && other) - { - this->swap(other); - std::swap(scale, other.scale); - return *this; - } - - UInt32 getScale() const { return scale; } - -private: - UInt32 scale; -}; - -/// Prevent implicit template instantiation of DecimalPaddedPODArray for common decimal types - -extern template class DecimalPaddedPODArray; -extern template class DecimalPaddedPODArray; -extern template class DecimalPaddedPODArray; -extern template class DecimalPaddedPODArray; -extern template class DecimalPaddedPODArray; - /// A ColumnVector for Decimals template class ColumnDecimal final : public COWHelper> @@ -72,16 +27,16 @@ private: public: using ValueType = T; using NativeT = typename T::NativeType; - using Container = DecimalPaddedPODArray; + using Container = PaddedPODArray; private: ColumnDecimal(const size_t n, UInt32 scale_) - : data(n, scale_), + : data(n), scale(scale_) {} ColumnDecimal(const ColumnDecimal & src) - : data(src.data), + : data(src.data.begin(), src.data.end()), scale(src.scale) {} @@ -195,7 +150,7 @@ public: const T & getElement(size_t n) const { return data[n]; } T & getElement(size_t n) { return data[n]; } - UInt32 getScale() const {return scale;} + UInt32 getScale() const { return scale; } protected: Container data; @@ -206,17 +161,17 @@ protected: { size_t s = data.size(); res.resize(s); - for (U i = 0; i < s; ++i) - res[i] = i; + for (size_t i = 0; i < s; ++i) + res[i] = static_cast(i); auto sort_end = res.end(); if (limit && limit < s) sort_end = res.begin() + limit; if (reverse) - partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; }); + ::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] > data[b]; }); else - partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; }); + ::partial_sort(res.begin(), sort_end, res.end(), [this](size_t a, size_t b) { return data[a] < data[b]; }); } }; diff --git a/src/Columns/ColumnFixedString.cpp b/src/Columns/ColumnFixedString.cpp index 0828f8ebd89..81eb9615ff9 100644 --- a/src/Columns/ColumnFixedString.cpp +++ b/src/Columns/ColumnFixedString.cpp @@ -192,9 +192,9 @@ void ColumnFixedString::getPermutation(bool reverse, size_t limit, int /*nan_dir else { if (reverse) - std::sort(res.begin(), res.end(), greater(*this)); + ::sort(res.begin(), res.end(), greater(*this)); else - std::sort(res.begin(), res.end(), less(*this)); + ::sort(res.begin(), res.end(), less(*this)); } } diff --git a/src/Columns/ColumnMap.cpp b/src/Columns/ColumnMap.cpp index e595525d9e8..ef5d96da0f7 100644 --- a/src/Columns/ColumnMap.cpp +++ b/src/Columns/ColumnMap.cpp @@ -4,8 +4,6 @@ #include #include #include -#include -#include #include #include #include @@ -64,8 +62,9 @@ MutableColumnPtr ColumnMap::cloneResized(size_t new_size) const Field ColumnMap::operator[](size_t n) const { - auto array = DB::get((*nested)[n]); - return Map(std::make_move_iterator(array.begin()), std::make_move_iterator(array.end())); + Field res; + get(n, res); + return res; } void ColumnMap::get(size_t n, Field & res) const @@ -74,11 +73,12 @@ void ColumnMap::get(size_t n, Field & res) const size_t offset = offsets[n - 1]; size_t size = offsets[n] - offsets[n - 1]; - res = Map(size); + res = Map(); auto & map = DB::get(res); + map.reserve(size); for (size_t i = 0; i < size; ++i) - getNestedData().get(offset + i, map[i]); + map.push_back(getNestedData()[offset + i]); } bool ColumnMap::isDefaultAt(size_t n) const diff --git a/src/Columns/ColumnString.cpp b/src/Columns/ColumnString.cpp index cd8a3e698d8..ef972059b00 100644 --- a/src/Columns/ColumnString.cpp +++ b/src/Columns/ColumnString.cpp @@ -335,9 +335,9 @@ void ColumnString::getPermutationImpl(size_t limit, Permutation & res, Comparato auto less = [&cmp](size_t lhs, size_t rhs){ return cmp(lhs, rhs) < 0; }; if (limit) - partial_sort(res.begin(), res.begin() + limit, res.end(), less); + ::partial_sort(res.begin(), res.begin() + limit, res.end(), less); else - std::sort(res.begin(), res.end(), less); + ::sort(res.begin(), res.end(), less); } void ColumnString::getPermutation(bool reverse, size_t limit, int /*nan_direction_hint*/, Permutation & res) const diff --git a/src/Columns/ColumnTuple.cpp b/src/Columns/ColumnTuple.cpp index d667b264d55..1b511f17f73 100644 --- a/src/Columns/ColumnTuple.cpp +++ b/src/Columns/ColumnTuple.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -9,9 +10,6 @@ #include #include #include -#include -#include -#include #include @@ -101,17 +99,21 @@ MutableColumnPtr ColumnTuple::cloneResized(size_t new_size) const Field ColumnTuple::operator[](size_t n) const { - return collections::map(columns, [n] (const auto & column) { return (*column)[n]; }); + Field res; + get(n, res); + return res; } void ColumnTuple::get(size_t n, Field & res) const { const size_t tuple_size = columns.size(); - Tuple tuple(tuple_size); - for (const auto i : collections::range(0, tuple_size)) - columns[i]->get(n, tuple[i]); - res = tuple; + res = Tuple(); + Tuple & res_tuple = DB::get(res); + res_tuple.reserve(tuple_size); + + for (size_t i = 0; i < tuple_size; ++i) + res_tuple.push_back((*columns[i])[n]); } bool ColumnTuple::isDefaultAt(size_t n) const @@ -383,9 +385,9 @@ void ColumnTuple::getPermutationImpl(size_t limit, Permutation & res, LessOperat limit = 0; if (limit) - partial_sort(res.begin(), res.begin() + limit, res.end(), less); + ::partial_sort(res.begin(), res.begin() + limit, res.end(), less); else - std::sort(res.begin(), res.end(), less); + ::sort(res.begin(), res.end(), less); } void ColumnTuple::updatePermutationImpl(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_ranges, const Collator * collator) const @@ -483,7 +485,7 @@ void ColumnTuple::getExtremes(Field & min, Field & max) const Tuple min_tuple(tuple_size); Tuple max_tuple(tuple_size); - for (const auto i : collections::range(0, tuple_size)) + for (size_t i = 0; i < tuple_size; ++i) columns[i]->getExtremes(min_tuple[i], max_tuple[i]); min = min_tuple; @@ -504,7 +506,7 @@ bool ColumnTuple::structureEquals(const IColumn & rhs) const if (tuple_size != rhs_tuple->columns.size()) return false; - for (const auto i : collections::range(0, tuple_size)) + for (size_t i = 0; i < tuple_size; ++i) if (!columns[i]->structureEquals(*rhs_tuple->columns[i])) return false; diff --git a/src/Columns/ColumnVector.cpp b/src/Columns/ColumnVector.cpp index 9808acf48c8..eca10049a0b 100644 --- a/src/Columns/ColumnVector.cpp +++ b/src/Columns/ColumnVector.cpp @@ -1,6 +1,5 @@ #include "ColumnVector.h" -#include #include #include #include @@ -118,7 +117,6 @@ struct ColumnVector::equals bool operator()(size_t lhs, size_t rhs) const { return CompareHelper::equals(parent.data[lhs], parent.data[rhs], nan_direction_hint); } }; - namespace { template @@ -158,9 +156,9 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi res[i] = i; if (reverse) - partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint)); + ::partial_sort(res.begin(), res.begin() + limit, res.end(), greater(*this, nan_direction_hint)); else - partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint)); + ::partial_sort(res.begin(), res.begin() + limit, res.end(), less(*this, nan_direction_hint)); } else { @@ -204,16 +202,16 @@ void ColumnVector::getPermutation(bool reverse, size_t limit, int nan_directi res[i] = i; if (reverse) - pdqsort(res.begin(), res.end(), greater(*this, nan_direction_hint)); + ::sort(res.begin(), res.end(), greater(*this, nan_direction_hint)); else - pdqsort(res.begin(), res.end(), less(*this, nan_direction_hint)); + ::sort(res.begin(), res.end(), less(*this, nan_direction_hint)); } } template void ColumnVector::updatePermutation(bool reverse, size_t limit, int nan_direction_hint, IColumn::Permutation & res, EqualRanges & equal_range) const { - auto sort = [](auto begin, auto end, auto pred) { pdqsort(begin, end, pred); }; + auto sort = [](auto begin, auto end, auto pred) { ::sort(begin, end, pred); }; auto partial_sort = [](auto begin, auto mid, auto end, auto pred) { ::partial_sort(begin, mid, end, pred); }; if (reverse) diff --git a/src/Columns/FilterDescription.cpp b/src/Columns/FilterDescription.cpp index 973d5bc4391..f8f4ee365ef 100644 --- a/src/Columns/FilterDescription.cpp +++ b/src/Columns/FilterDescription.cpp @@ -91,4 +91,14 @@ FilterDescription::FilterDescription(const IColumn & column_) ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER); } +SparseFilterDescription::SparseFilterDescription(const IColumn & column) +{ + const auto * column_sparse = typeid_cast(&column); + if (!column_sparse || !typeid_cast(&column_sparse->getValuesColumn())) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_COLUMN_FOR_FILTER, + "Illegal type {} of column for sparse filter. Must be Sparse(UInt8)", column.getName()); + + filter_indices = &column_sparse->getOffsetsColumn(); +} + } diff --git a/src/Columns/FilterDescription.h b/src/Columns/FilterDescription.h index 05812fea283..18c95d24b9d 100644 --- a/src/Columns/FilterDescription.h +++ b/src/Columns/FilterDescription.h @@ -1,6 +1,7 @@ #pragma once #include +#include namespace DB @@ -15,20 +16,37 @@ struct ConstantFilterDescription bool always_false = false; bool always_true = false; - ConstantFilterDescription() {} + ConstantFilterDescription() = default; explicit ConstantFilterDescription(const IColumn & column); }; +struct IFilterDescription +{ + virtual ColumnPtr filter(const IColumn & column, ssize_t result_size_hint) const = 0; + virtual size_t countBytesInFilter() const = 0; + virtual ~IFilterDescription() = default; +}; /// Obtain a filter from non constant Column, that may have type: UInt8, Nullable(UInt8). -struct FilterDescription +struct FilterDescription final : public IFilterDescription { const IColumn::Filter * data = nullptr; /// Pointer to filter when it is not always true or always false. ColumnPtr data_holder; /// If new column was generated, it will be owned by holder. explicit FilterDescription(const IColumn & column); + + ColumnPtr filter(const IColumn & column, ssize_t result_size_hint) const override { return column.filter(*data, result_size_hint); } + size_t countBytesInFilter() const override { return DB::countBytesInFilter(*data); } }; +struct SparseFilterDescription final : public IFilterDescription +{ + const IColumn * filter_indices = nullptr; + explicit SparseFilterDescription(const IColumn & column); + + ColumnPtr filter(const IColumn & column, ssize_t) const override { return column.index(*filter_indices, 0); } + size_t countBytesInFilter() const override { return filter_indices->size(); } +}; struct ColumnWithTypeAndName; diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h index b1a6e83ee98..303c78506c4 100644 --- a/src/Columns/IColumn.h +++ b/src/Columns/IColumn.h @@ -528,7 +528,7 @@ protected: template void getIndicesOfNonDefaultRowsImpl(Offsets & indices, size_t from, size_t limit) const; - /// Uses std::sort and partial_sort as default algorithms. + /// Uses sort and partial_sort as default algorithms. /// Implements 'less' and 'equals' via comparator. /// If 'less' and 'equals' can be implemented more optimal /// (e.g. with less number of comparisons), you can use diff --git a/src/Columns/IColumnImpl.h b/src/Columns/IColumnImpl.h index 1be52087d11..450684aacea 100644 --- a/src/Columns/IColumnImpl.h +++ b/src/Columns/IColumnImpl.h @@ -11,6 +11,7 @@ #include #include + namespace DB { namespace ErrorCodes @@ -203,7 +204,7 @@ void IColumn::updatePermutationImpl( limit, res, equal_ranges, [&cmp](size_t lhs, size_t rhs) { return cmp(lhs, rhs) < 0; }, [&cmp](size_t lhs, size_t rhs) { return cmp(lhs, rhs) == 0; }, - [](auto begin, auto end, auto pred) { std::sort(begin, end, pred); }, + [](auto begin, auto end, auto pred) { ::sort(begin, end, pred); }, [](auto begin, auto mid, auto end, auto pred) { ::partial_sort(begin, mid, end, pred); }); } diff --git a/src/Common/ArenaUtils.h b/src/Common/ArenaUtils.h new file mode 100644 index 00000000000..0a588692367 --- /dev/null +++ b/src/Common/ArenaUtils.h @@ -0,0 +1,20 @@ +#pragma once + +#include +#include +#include + +/** Copy string value into Arena. + * Arena should support method: + * char * alloc(size_t size). + */ +template +inline StringRef copyStringInArena(Arena & arena, StringRef value) +{ + size_t key_size = value.size; + char * place_for_key = arena.alloc(key_size); + memcpy(reinterpret_cast(place_for_key), reinterpret_cast(value.data), key_size); + StringRef result{place_for_key, key_size}; + + return result; +} diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index f32707798f7..fbd3e71f9b8 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -387,47 +387,52 @@ struct HashMethodSingleLowCardinalityColumn : public SingleColumnMethod } template - ALWAYS_INLINE FindResult findFromRow(Data & data, size_t row_, Arena & pool) + ALWAYS_INLINE FindResult findKey(Data & data, size_t row_, Arena & pool) { size_t row = getIndexAt(row_); if (is_nullable && row == 0) { if constexpr (has_mapped) - return FindResult(data.hasNullKeyData() ? &data.getNullKeyData() : nullptr, data.hasNullKeyData()); + return FindResult(data.hasNullKeyData() ? &data.getNullKeyData() : nullptr, data.hasNullKeyData(), 0); else - return FindResult(data.hasNullKeyData()); + return FindResult(data.hasNullKeyData(), 0); } if (visit_cache[row] != VisitValue::Empty) { if constexpr (has_mapped) - return FindResult(&mapped_cache[row], visit_cache[row] == VisitValue::Found); + return FindResult(&mapped_cache[row], visit_cache[row] == VisitValue::Found, 0); else - return FindResult(visit_cache[row] == VisitValue::Found); + return FindResult(visit_cache[row] == VisitValue::Found, 0); } auto key_holder = getKeyHolder(row_, pool); - typename Data::iterator it; + typename Data::LookupResult it; if (saved_hash) - it = data.find(*key_holder, saved_hash[row]); + it = data.find(keyHolderGetKey(key_holder), saved_hash[row]); else - it = data.find(*key_holder); + it = data.find(keyHolderGetKey(key_holder)); - bool found = it != data.end(); + bool found = it; visit_cache[row] = found ? VisitValue::Found : VisitValue::NotFound; if constexpr (has_mapped) { if (found) - mapped_cache[row] = it->second; + mapped_cache[row] = it->getMapped(); } + size_t offset = 0; + + if constexpr (FindResult::has_offset) + offset = found ? data.offsetInternal(it) : 0; + if constexpr (has_mapped) - return FindResult(&mapped_cache[row], found); + return FindResult(&mapped_cache[row], found, offset); else - return FindResult(found); + return FindResult(found, offset); } template diff --git a/src/Common/Config/ConfigProcessor.cpp b/src/Common/Config/ConfigProcessor.cpp index 41e9a53e50f..b86e8ed3e40 100644 --- a/src/Common/Config/ConfigProcessor.cpp +++ b/src/Common/Config/ConfigProcessor.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include @@ -105,7 +106,7 @@ static ElementIdentifier getElementIdentifier(Node * element) std::string value = node->nodeValue(); attrs_kv.push_back(std::make_pair(name, value)); } - std::sort(attrs_kv.begin(), attrs_kv.end()); + ::sort(attrs_kv.begin(), attrs_kv.end()); ElementIdentifier res; res.push_back(element->nodeName()); @@ -443,7 +444,7 @@ ConfigProcessor::Files ConfigProcessor::getConfigMergeFiles(const std::string & } } - std::sort(files.begin(), files.end()); + ::sort(files.begin(), files.end()); return files; } diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 896168253cf..a741f1f1bfc 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -80,6 +80,7 @@ M(SyncDrainedConnections, "Number of connections drained synchronously.") \ M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \ M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \ + M(PendingAsyncInsert, "Number of asynchronous inserts that are waiting for flush.") \ namespace CurrentMetrics { diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index 36d0c13b153..13da3efd57a 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -272,7 +272,7 @@ bool DNSResolver::updateCacheImpl(UpdateF && update_func, ElemsT && elems, const } if (!lost_elems.empty()) - LOG_INFO(log, log_msg, lost_elems); + LOG_INFO(log, fmt::runtime(log_msg), lost_elems); return updated; } diff --git a/src/Common/ErrorCodes.cpp b/src/Common/ErrorCodes.cpp index 82714de3470..e991daf3209 100644 --- a/src/Common/ErrorCodes.cpp +++ b/src/Common/ErrorCodes.cpp @@ -610,6 +610,8 @@ M(639, SNAPPY_COMPRESS_FAILED) \ M(640, NO_HIVEMETASTORE) \ M(641, CANNOT_APPEND_TO_FILE) \ + M(642, CANNOT_PACK_ARCHIVE) \ + M(643, CANNOT_UNPACK_ARCHIVE) \ \ M(999, KEEPER_EXCEPTION) \ M(1000, POCO_EXCEPTION) \ diff --git a/src/Common/Exception.h b/src/Common/Exception.h index 3aa06f8c988..b6bc31a5821 100644 --- a/src/Common/Exception.h +++ b/src/Common/Exception.h @@ -37,7 +37,7 @@ public: // Format message with fmt::format, like the logging functions. template Exception(int code, const std::string & fmt, Args&&... args) - : Exception(fmt::format(fmt, std::forward(args)...), code) + : Exception(fmt::format(fmt::runtime(fmt), std::forward(args)...), code) {} struct CreateFromPocoTag {}; @@ -55,7 +55,7 @@ public: template void addMessage(const std::string& format, Args&&... args) { - extendedMessage(fmt::format(format, std::forward(args)...)); + extendedMessage(fmt::format(fmt::runtime(format), std::forward(args)...)); } void addMessage(const std::string& message) @@ -119,7 +119,7 @@ public: // Format message with fmt::format, like the logging functions. template ParsingException(int code, const std::string & fmt, Args&&... args) - : Exception(fmt::format(fmt, std::forward(args)...), code) + : Exception(fmt::format(fmt::runtime(fmt), std::forward(args)...), code) {} diff --git a/src/Common/IntervalTree.h b/src/Common/IntervalTree.h index 608bd5cf12c..b8334653754 100644 --- a/src/Common/IntervalTree.h +++ b/src/Common/IntervalTree.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -489,14 +490,14 @@ private: } } - std::sort(intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end(), [](auto & lhs, auto & rhs) + ::sort(intervals_sorted_by_left_asc.begin(), intervals_sorted_by_left_asc.end(), [](auto & lhs, auto & rhs) { auto & lhs_interval = getInterval(lhs); auto & rhs_interval = getInterval(rhs); return lhs_interval.left < rhs_interval.left; }); - std::sort(intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end(), [](auto & lhs, auto & rhs) + ::sort(intervals_sorted_by_right_desc.begin(), intervals_sorted_by_right_desc.end(), [](auto & lhs, auto & rhs) { auto & lhs_interval = getInterval(lhs); auto & rhs_interval = getInterval(rhs); @@ -681,7 +682,7 @@ private: size_t size = points.size(); size_t middle_element_index = size / 2; - std::nth_element(points.begin(), points.begin() + middle_element_index, points.end()); + ::nth_element(points.begin(), points.begin() + middle_element_index, points.end()); /** We should not get median as average of middle_element_index and middle_element_index - 1 * because we want point in node to intersect some interval. diff --git a/src/Common/LockMemoryExceptionInThread.h b/src/Common/LockMemoryExceptionInThread.h index dc2bccf257b..ec8f69806d7 100644 --- a/src/Common/LockMemoryExceptionInThread.h +++ b/src/Common/LockMemoryExceptionInThread.h @@ -1,5 +1,6 @@ #pragma once +#include #include /// To be able to avoid MEMORY_LIMIT_EXCEEDED Exception in destructors: diff --git a/src/Common/MemoryTracker.cpp b/src/Common/MemoryTracker.cpp index ba98ede221a..4c8af23791e 100644 --- a/src/Common/MemoryTracker.cpp +++ b/src/Common/MemoryTracker.cpp @@ -1,6 +1,7 @@ #include "MemoryTracker.h" #include +#include #include #include #include @@ -8,6 +9,7 @@ #include #include #include +#include #include #include @@ -95,7 +97,7 @@ void MemoryTracker::logMemoryUsage(Int64 current) const } -void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded) +void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryTracker * query_tracker) { if (size < 0) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Negative size ({}) is passed to MemoryTracker. It is a bug.", size); @@ -104,7 +106,8 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded) { /// Since the MemoryTrackerBlockerInThread should respect the level, we should go to the next parent. if (auto * loaded_next = parent.load(std::memory_order_relaxed)) - loaded_next->allocImpl(size, throw_if_memory_exceeded); + loaded_next->allocImpl(size, throw_if_memory_exceeded, + level == VariableContext::Process ? this : query_tracker); return; } @@ -186,18 +189,30 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded) if (unlikely(current_hard_limit && will_be > current_hard_limit) && memoryTrackerCanThrow(level, false) && throw_if_memory_exceeded) { - /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc - MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global); - ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded); - const auto * description = description_ptr.load(std::memory_order_relaxed); - throw DB::Exception( - DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, - "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}", - description ? " " : "", - description ? description : "", - formatReadableSizeWithBinarySuffix(will_be), - size, - formatReadableSizeWithBinarySuffix(current_hard_limit)); + bool need_to_throw = true; + bool try_to_free_memory = overcommit_tracker != nullptr && query_tracker != nullptr; + if (try_to_free_memory) + need_to_throw = overcommit_tracker->needToStopQuery(query_tracker); + + if (need_to_throw) + { + /// Prevent recursion. Exception::ctor -> std::string -> new[] -> MemoryTracker::alloc + MemoryTrackerBlockerInThread untrack_lock(VariableContext::Global); + ProfileEvents::increment(ProfileEvents::QueryMemoryLimitExceeded); + const auto * description = description_ptr.load(std::memory_order_relaxed); + throw DB::Exception( + DB::ErrorCodes::MEMORY_LIMIT_EXCEEDED, + "Memory limit{}{} exceeded: would use {} (attempt to allocate chunk of {} bytes), maximum: {}", + description ? " " : "", + description ? description : "", + formatReadableSizeWithBinarySuffix(will_be), + size, + formatReadableSizeWithBinarySuffix(current_hard_limit)); + } + else + { + will_be = amount.load(std::memory_order_relaxed); + } } bool peak_updated; @@ -221,7 +236,8 @@ void MemoryTracker::allocImpl(Int64 size, bool throw_if_memory_exceeded) } if (auto * loaded_next = parent.load(std::memory_order_relaxed)) - loaded_next->allocImpl(size, throw_if_memory_exceeded); + loaded_next->allocImpl(size, throw_if_memory_exceeded, + level == VariableContext::Process ? this : query_tracker); } void MemoryTracker::alloc(Int64 size) @@ -302,10 +318,23 @@ void MemoryTracker::free(Int64 size) } +OvercommitRatio MemoryTracker::getOvercommitRatio() +{ + return { amount.load(std::memory_order_relaxed), soft_limit.load(std::memory_order_relaxed) }; +} + + +OvercommitRatio MemoryTracker::getOvercommitRatio(Int64 limit) +{ + return { amount.load(std::memory_order_relaxed), limit }; +} + + void MemoryTracker::resetCounters() { amount.store(0, std::memory_order_relaxed); peak.store(0, std::memory_order_relaxed); + soft_limit.store(0, std::memory_order_relaxed); hard_limit.store(0, std::memory_order_relaxed); profiler_limit.store(0, std::memory_order_relaxed); } @@ -330,6 +359,12 @@ void MemoryTracker::set(Int64 to) } +void MemoryTracker::setSoftLimit(Int64 value) +{ + soft_limit.store(value, std::memory_order_relaxed); +} + + void MemoryTracker::setHardLimit(Int64 value) { hard_limit.store(value, std::memory_order_relaxed); diff --git a/src/Common/MemoryTracker.h b/src/Common/MemoryTracker.h index a0138b25b5f..780a869acad 100644 --- a/src/Common/MemoryTracker.h +++ b/src/Common/MemoryTracker.h @@ -28,6 +28,9 @@ extern thread_local bool memory_tracker_always_throw_logical_error_on_allocation #define ALLOW_ALLOCATIONS_IN_SCOPE static_assert(true) #endif +struct OvercommitRatio; +struct OvercommitTracker; + /** Tracks memory consumption. * It throws an exception if amount of consumed memory become greater than certain limit. * The same memory tracker could be simultaneously used in different threads. @@ -40,6 +43,7 @@ class MemoryTracker private: std::atomic amount {0}; std::atomic peak {0}; + std::atomic soft_limit {0}; std::atomic hard_limit {0}; std::atomic profiler_limit {0}; @@ -61,6 +65,8 @@ private: /// This description will be used as prefix into log messages (if isn't nullptr) std::atomic description_ptr = nullptr; + OvercommitTracker * overcommit_tracker = nullptr; + bool updatePeak(Int64 will_be, bool log_memory_usage); void logMemoryUsage(Int64 current) const; @@ -83,7 +89,7 @@ public: void allocNoThrow(Int64 size); - void allocImpl(Int64 size, bool throw_if_memory_exceeded); + void allocImpl(Int64 size, bool throw_if_memory_exceeded, MemoryTracker * query_tracker = nullptr); void realloc(Int64 old_size, Int64 new_size) { @@ -108,8 +114,14 @@ public: return peak.load(std::memory_order_relaxed); } + void setSoftLimit(Int64 value); void setHardLimit(Int64 value); + Int64 getSoftLimit() const + { + return soft_limit.load(std::memory_order_relaxed); + } + /** Set limit if it was not set. * Otherwise, set limit to new value, if new value is greater than previous limit. */ @@ -159,6 +171,14 @@ public: description_ptr.store(description, std::memory_order_relaxed); } + OvercommitRatio getOvercommitRatio(); + OvercommitRatio getOvercommitRatio(Int64 limit); + + void setOvercommitTracker(OvercommitTracker * tracker) noexcept + { + overcommit_tracker = tracker; + } + /// Reset the accumulated data void resetCounters(); diff --git a/src/Common/MemoryTrackerBlockerInThread.h b/src/Common/MemoryTrackerBlockerInThread.h index caad28f636e..381eb80df0c 100644 --- a/src/Common/MemoryTrackerBlockerInThread.h +++ b/src/Common/MemoryTrackerBlockerInThread.h @@ -1,5 +1,6 @@ #pragma once +#include #include /// To be able to temporarily stop memory tracking from current thread. diff --git a/src/Common/OvercommitTracker.cpp b/src/Common/OvercommitTracker.cpp new file mode 100644 index 00000000000..4be7096aa60 --- /dev/null +++ b/src/Common/OvercommitTracker.cpp @@ -0,0 +1,119 @@ +#include "OvercommitTracker.h" + +#include +#include +#include + +using namespace std::chrono_literals; + +OvercommitTracker::OvercommitTracker() + : max_wait_time(0us) + , picked_tracker(nullptr) + , cancelation_state(QueryCancelationState::NONE) +{} + +void OvercommitTracker::setMaxWaitTime(UInt64 wait_time) +{ + std::lock_guard guard(overcommit_m); + max_wait_time = wait_time * 1us; +} + +bool OvercommitTracker::needToStopQuery(MemoryTracker * tracker) +{ + std::unique_lock lk(overcommit_m); + + pickQueryToExclude(); + assert(cancelation_state == QueryCancelationState::RUNNING); + + // If no query was chosen we need to stop current query. + // This may happen if no soft limit is set. + if (picked_tracker == nullptr) + { + cancelation_state = QueryCancelationState::NONE; + return true; + } + if (picked_tracker == tracker) + return true; + return !cv.wait_for(lk, max_wait_time, [this]() + { + return cancelation_state == QueryCancelationState::NONE; + }); +} + +void OvercommitTracker::unsubscribe(MemoryTracker * tracker) +{ + std::unique_lock lk(overcommit_m); + if (picked_tracker == tracker) + { + LOG_DEBUG(getLogger(), "Picked query stopped"); + + picked_tracker = nullptr; + cancelation_state = QueryCancelationState::NONE; + cv.notify_all(); + } +} + +UserOvercommitTracker::UserOvercommitTracker(DB::ProcessListForUser * user_process_list_) + : user_process_list(user_process_list_) +{} + +void UserOvercommitTracker::pickQueryToExcludeImpl() +{ + MemoryTracker * query_tracker = nullptr; + OvercommitRatio current_ratio{0, 0}; + // At this moment query list must be read only. + // BlockQueryIfMemoryLimit is used in ProcessList to guarantee this. + auto & queries = user_process_list->queries; + LOG_DEBUG(logger, "Trying to choose query to stop from {} queries", queries.size()); + for (auto const & query : queries) + { + if (query.second->isKilled()) + continue; + + auto * memory_tracker = query.second->getMemoryTracker(); + if (!memory_tracker) + continue; + + auto ratio = memory_tracker->getOvercommitRatio(); + LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); + if (ratio.soft_limit != 0 && current_ratio < ratio) + { + query_tracker = memory_tracker; + current_ratio = ratio; + } + } + LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}", + current_ratio.committed, current_ratio.soft_limit); + picked_tracker = query_tracker; +} + +void GlobalOvercommitTracker::pickQueryToExcludeImpl() +{ + MemoryTracker * query_tracker = nullptr; + OvercommitRatio current_ratio{0, 0}; + process_list->processEachQueryStatus([&](DB::QueryStatus const & query) + { + if (query.isKilled()) + return; + + Int64 user_soft_limit = 0; + if (auto const * user_process_list = query.getUserProcessList()) + user_soft_limit = user_process_list->user_memory_tracker.getSoftLimit(); + if (user_soft_limit == 0) + return; + + auto * memory_tracker = query.getMemoryTracker(); + if (!memory_tracker) + return; + auto ratio = memory_tracker->getOvercommitRatio(user_soft_limit); + LOG_DEBUG(logger, "Query has ratio {}/{}", ratio.committed, ratio.soft_limit); + if (current_ratio < ratio) + { + query_tracker = memory_tracker; + current_ratio = ratio; + } + }); + LOG_DEBUG(logger, "Selected to stop query with overcommit ratio {}/{}", + current_ratio.committed, current_ratio.soft_limit); + picked_tracker = query_tracker; +} diff --git a/src/Common/OvercommitTracker.h b/src/Common/OvercommitTracker.h new file mode 100644 index 00000000000..2286ad4bde2 --- /dev/null +++ b/src/Common/OvercommitTracker.h @@ -0,0 +1,155 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +// This struct is used for the comparison of query memory usage. +struct OvercommitRatio +{ + OvercommitRatio(Int64 committed_, Int64 soft_limit_) + : committed(committed_) + , soft_limit(soft_limit_) + {} + + friend bool operator<(OvercommitRatio const & lhs, OvercommitRatio const & rhs) noexcept + { + // (a / b < c / d) <=> (a * d < c * b) + return (lhs.committed * rhs.soft_limit) < (rhs.committed * lhs.soft_limit) + || (lhs.soft_limit == 0 && rhs.soft_limit > 0) + || (lhs.committed == 0 && rhs.committed == 0 && lhs.soft_limit > rhs.soft_limit); + } + + // actual query memory usage + Int64 committed; + // guaranteed amount of memory query can use + Int64 soft_limit; +}; + +class MemoryTracker; + +// Usually it's hard to set some reasonable hard memory limit +// (especially, the default value). This class introduces new +// mechanisim for the limiting of memory usage. +// Soft limit represents guaranteed amount of memory query/user +// may use. It's allowed to exceed this limit. But if hard limit +// is reached, query with the biggest overcommit ratio +// is killed to free memory. +struct OvercommitTracker : boost::noncopyable +{ + OvercommitTracker(); + + void setMaxWaitTime(UInt64 wait_time); + + bool needToStopQuery(MemoryTracker * tracker); + + void unsubscribe(MemoryTracker * tracker); + + virtual ~OvercommitTracker() = default; + +protected: + virtual void pickQueryToExcludeImpl() = 0; + + mutable std::mutex overcommit_m; + mutable std::condition_variable cv; + + std::chrono::microseconds max_wait_time; + + enum class QueryCancelationState + { + NONE, + RUNNING, + }; + + // Specifies memory tracker of the chosen to stop query. + // If soft limit is not set, all the queries which reach hard limit must stop. + // This case is represented as picked tracker pointer is set to nullptr and + // overcommit tracker is in RUNNING state. + MemoryTracker * picked_tracker; + QueryCancelationState cancelation_state; + + virtual Poco::Logger * getLogger() = 0; + +private: + + void pickQueryToExclude() + { + if (cancelation_state != QueryCancelationState::RUNNING) + { + pickQueryToExcludeImpl(); + cancelation_state = QueryCancelationState::RUNNING; + } + } + + friend struct BlockQueryIfMemoryLimit; +}; + +namespace DB +{ + class ProcessList; + struct ProcessListForUser; +} + +struct UserOvercommitTracker : OvercommitTracker +{ + explicit UserOvercommitTracker(DB::ProcessListForUser * user_process_list_); + + ~UserOvercommitTracker() override = default; + +protected: + void pickQueryToExcludeImpl() override final; + + Poco::Logger * getLogger() override final { return logger; } +private: + DB::ProcessListForUser * user_process_list; + Poco::Logger * logger = &Poco::Logger::get("UserOvercommitTracker"); +}; + +struct GlobalOvercommitTracker : OvercommitTracker +{ + explicit GlobalOvercommitTracker(DB::ProcessList * process_list_) + : process_list(process_list_) + {} + + ~GlobalOvercommitTracker() override = default; + +protected: + void pickQueryToExcludeImpl() override final; + + Poco::Logger * getLogger() override final { return logger; } +private: + DB::ProcessList * process_list; + Poco::Logger * logger = &Poco::Logger::get("GlobalOvercommitTracker"); +}; + +// UserOvercommitTracker requires to check the whole list of user's queries +// to pick one to stop. BlockQueryIfMemoryLimit struct allows to wait until +// query selection is finished. It's used in ProcessList to make user query +// list immutable when UserOvercommitTracker reads it. +struct BlockQueryIfMemoryLimit +{ + BlockQueryIfMemoryLimit(OvercommitTracker const & overcommit_tracker) + : mutex(overcommit_tracker.overcommit_m) + , lk(mutex) + { + if (overcommit_tracker.cancelation_state == OvercommitTracker::QueryCancelationState::RUNNING) + { + overcommit_tracker.cv.wait_for(lk, overcommit_tracker.max_wait_time, [&overcommit_tracker]() + { + return overcommit_tracker.cancelation_state == OvercommitTracker::QueryCancelationState::NONE; + }); + } + } + + ~BlockQueryIfMemoryLimit() = default; + +private: + std::mutex & mutex; + std::unique_lock lk; +}; diff --git a/src/Common/PoolBase.h b/src/Common/PoolBase.h index 85d4e84abca..a82a6efc4c1 100644 --- a/src/Common/PoolBase.h +++ b/src/Common/PoolBase.h @@ -41,6 +41,7 @@ private: ObjectPtr object; bool in_use = false; + std::atomic is_expired = false; PoolBase & pool; }; @@ -87,6 +88,14 @@ public: Object & operator*() & { return *data->data.object; } const Object & operator*() const & { return *data->data.object; } + /** + * Expire an object to make it reallocated later. + */ + void expire() + { + data->data.is_expired = true; + } + bool isNull() const { return data == nullptr; } PoolBase * getPool() const @@ -112,9 +121,22 @@ public: while (true) { for (auto & item : items) + { if (!item->in_use) - return Entry(*item); - + { + if (likely(!item->is_expired)) + { + return Entry(*item); + } + else + { + expireObject(item->object); + item->object = allocObject(); + item->is_expired = false; + return Entry(*item); + } + } + } if (items.size() < max_items) { ObjectPtr object = allocObject(); @@ -139,6 +161,12 @@ public: items.emplace_back(std::make_shared(allocObject(), *this)); } + inline size_t size() + { + std::unique_lock lock(mutex); + return items.size(); + } + private: /** The maximum size of the pool. */ unsigned max_items; @@ -162,4 +190,5 @@ protected: /** Creates a new object to put into the pool. */ virtual ObjectPtr allocObject() = 0; + virtual void expireObject(ObjectPtr) {} }; diff --git a/src/Common/PoolWithFailoverBase.h b/src/Common/PoolWithFailoverBase.h index b8fa00d2703..ae14011834a 100644 --- a/src/Common/PoolWithFailoverBase.h +++ b/src/Common/PoolWithFailoverBase.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -178,7 +179,7 @@ PoolWithFailoverBase::getShuffledPools( shuffled_pools.reserve(nested_pools.size()); for (size_t i = 0; i < nested_pools.size(); ++i) shuffled_pools.push_back(ShuffledPool{nested_pools[i].get(), &pool_states[i], i, 0}); - std::sort( + ::sort( shuffled_pools.begin(), shuffled_pools.end(), [](const ShuffledPool & lhs, const ShuffledPool & rhs) { diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index ea6c782ebb4..1e7d1b81b37 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -8,6 +8,7 @@ M(Query, "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.") \ M(SelectQuery, "Same as Query, but only for SELECT queries.") \ M(InsertQuery, "Same as Query, but only for INSERT queries.") \ + M(AsyncInsertQuery, "Same as InsertQuery, but only for asynchronous INSERT queries.") \ M(FailedQuery, "Number of failed queries.") \ M(FailedSelectQuery, "Same as FailedQuery, but only for SELECT queries.") \ M(FailedInsertQuery, "Same as FailedQuery, but only for INSERT queries.") \ @@ -280,6 +281,10 @@ M(ExternalDataSourceLocalCacheReadBytes, "Bytes read from local cache buffer in RemoteReadBufferCache")\ \ M(MainConfigLoads, "Number of times the main configuration was reloaded.") \ + \ + M(ScalarSubqueriesGlobalCacheHit, "Number of times a read from a scalar subquery was done using the global cache") \ + M(ScalarSubqueriesLocalCacheHit, "Number of times a read from a scalar subquery was done using the local cache") \ + M(ScalarSubqueriesCacheMiss, "Number of times a read from a scalar subquery was not cached and had to be calculated completely") namespace ProfileEvents { diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index b9a8bc923f7..00e2326b0b4 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -243,7 +243,7 @@ void ProgressIndication::writeProgress() if (width_of_progress_bar > 0) { - size_t bar_width = UnicodeBar::getWidth(current_count, 0, max_count, width_of_progress_bar); + double bar_width = UnicodeBar::getWidth(current_count, 0, max_count, width_of_progress_bar); std::string bar = UnicodeBar::render(bar_width); /// Render profiling_msg at left on top of the progress bar. diff --git a/src/Common/SpaceSaving.h b/src/Common/SpaceSaving.h index d1e6d079d17..03d2c9638a4 100644 --- a/src/Common/SpaceSaving.h +++ b/src/Common/SpaceSaving.h @@ -5,6 +5,8 @@ #include +#include + #include #include #include @@ -242,7 +244,7 @@ public: } } - std::sort(counter_list.begin(), counter_list.end(), [](Counter * l, Counter * r) { return *l > *r; }); + ::sort(counter_list.begin(), counter_list.end(), [](Counter * l, Counter * r) { return *l > *r; }); if (counter_list.size() > m_capacity) { diff --git a/src/Common/SymbolIndex.cpp b/src/Common/SymbolIndex.cpp index 32c1a15337c..af718ea77b9 100644 --- a/src/Common/SymbolIndex.cpp +++ b/src/Common/SymbolIndex.cpp @@ -12,6 +12,8 @@ //#include #include +#include + /** ELF object can contain three different places with symbol names and addresses: @@ -498,8 +500,8 @@ void SymbolIndex::update() { dl_iterate_phdr(collectSymbols, &data); - std::sort(data.objects.begin(), data.objects.end(), [](const Object & a, const Object & b) { return a.address_begin < b.address_begin; }); - std::sort(data.symbols.begin(), data.symbols.end(), [](const Symbol & a, const Symbol & b) { return a.address_begin < b.address_begin; }); + ::sort(data.objects.begin(), data.objects.end(), [](const Object & a, const Object & b) { return a.address_begin < b.address_begin; }); + ::sort(data.symbols.begin(), data.symbols.end(), [](const Symbol & a, const Symbol & b) { return a.address_begin < b.address_begin; }); /// We found symbols both from loaded program headers and from ELF symbol tables. data.symbols.erase(std::unique(data.symbols.begin(), data.symbols.end(), [](const Symbol & a, const Symbol & b) diff --git a/src/Common/ThreadPool.cpp b/src/Common/ThreadPool.cpp index 9b01987c7cf..8bfb93c9e94 100644 --- a/src/Common/ThreadPool.cpp +++ b/src/Common/ThreadPool.cpp @@ -54,6 +54,9 @@ void ThreadPoolImpl::setMaxThreads(size_t value) { std::lock_guard lock(mutex); max_threads = value; + /// We have to also adjust queue size, because it limits the number of scheduled and already running jobs in total. + queue_size = std::max(queue_size, max_threads); + jobs.reserve(queue_size); } template diff --git a/src/Common/ZooKeeper/ZooKeeper.cpp b/src/Common/ZooKeeper/ZooKeeper.cpp index c8753c8edaf..b1574341c40 100644 --- a/src/Common/ZooKeeper/ZooKeeper.cpp +++ b/src/Common/ZooKeeper/ZooKeeper.cpp @@ -1145,7 +1145,7 @@ std::string normalizeZooKeeperPath(std::string zookeeper_path, bool check_starts if (check_starts_with_slash) throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "ZooKeeper path must starts with '/', got '{}'", zookeeper_path); if (log) - LOG_WARNING(log, "ZooKeeper path ('{}') does not start with '/'. It will not be supported in future releases"); + LOG_WARNING(log, "ZooKeeper path ('{}') does not start with '/'. It will not be supported in future releases", zookeeper_path); zookeeper_path = "/" + zookeeper_path; } diff --git a/src/Common/config.h.in b/src/Common/config.h.in index 3d785e0d0fb..edade4ce2be 100644 --- a/src/Common/config.h.in +++ b/src/Common/config.h.in @@ -17,6 +17,7 @@ #cmakedefine01 USE_YAML_CPP #cmakedefine01 CLICKHOUSE_SPLIT_BINARY #cmakedefine01 USE_BZIP2 +#cmakedefine01 USE_MINIZIP #cmakedefine01 USE_SNAPPY #cmakedefine01 USE_HIVE #cmakedefine01 USE_ODBC diff --git a/src/Common/tests/gtest_log.cpp b/src/Common/tests/gtest_log.cpp index 5addb5acf5d..b25f1cf117a 100644 --- a/src/Common/tests/gtest_log.cpp +++ b/src/Common/tests/gtest_log.cpp @@ -17,7 +17,7 @@ TEST(Logger, Log) Poco::Logger * log = &Poco::Logger::get("Log"); /// This test checks that we don't pass this string to fmtlib, because it is the only argument. - EXPECT_NO_THROW(LOG_INFO(log, "Hello {} World")); + EXPECT_NO_THROW(LOG_INFO(log, fmt::runtime("Hello {} World"))); } TEST(Logger, TestLog) diff --git a/src/Common/tests/gtest_poolbase.cpp b/src/Common/tests/gtest_poolbase.cpp new file mode 100644 index 00000000000..20c3281c964 --- /dev/null +++ b/src/Common/tests/gtest_poolbase.cpp @@ -0,0 +1,52 @@ +#include +#include +#include +#include +using namespace DB; + +class PoolObject +{ +public: + int x = 0; +}; + +class MyPoolBase : public PoolBase +{ +public: + using Object = PoolBase::Object; + using ObjectPtr = std::shared_ptr; + using Ptr = PoolBase::Ptr; + + int last_destroy_value = 0; + MyPoolBase() : PoolBase(100, &Poco::Logger::get("MyPoolBase")) { } + +protected: + ObjectPtr allocObject() override { return std::make_shared(); } + + void expireObject(ObjectPtr obj) override + { + LOG_TRACE(log, "expire object"); + ASSERT_TRUE(obj->x == 100); + last_destroy_value = obj->x; + } +}; + +TEST(PoolBase, testDestroy1) +{ + MyPoolBase pool; + { + auto obj_entry = pool.get(-1); + ASSERT_TRUE(!obj_entry.isNull()); + obj_entry->x = 100; + obj_entry.expire(); + } + ASSERT_EQ(1, pool.size()); + + { + auto obj_entry = pool.get(-1); + ASSERT_TRUE(!obj_entry.isNull()); + ASSERT_EQ(obj_entry->x, 0); + ASSERT_EQ(1, pool.size()); + } + ASSERT_EQ(100, pool.last_destroy_value); +} diff --git a/src/Compression/CachedCompressedReadBuffer.cpp b/src/Compression/CachedCompressedReadBuffer.cpp index f942f81f5e9..7f6422ae734 100644 --- a/src/Compression/CachedCompressedReadBuffer.cpp +++ b/src/Compression/CachedCompressedReadBuffer.cpp @@ -30,6 +30,7 @@ void CachedCompressedReadBuffer::initInput() void CachedCompressedReadBuffer::prefetch() { + initInput(); file_in->prefetch(); } @@ -105,7 +106,7 @@ void CachedCompressedReadBuffer::seek(size_t offset_in_compressed_file, size_t o /// We will discard our working_buffer, but have to account rest bytes bytes += offset(); /// No data, everything discarded - pos = working_buffer.end(); + resetWorkingBuffer(); owned_cell.reset(); /// Remember required offset in decompressed block which will be set in diff --git a/src/Compression/CachedCompressedReadBuffer.h b/src/Compression/CachedCompressedReadBuffer.h index 16770e343cc..8394c478319 100644 --- a/src/Compression/CachedCompressedReadBuffer.h +++ b/src/Compression/CachedCompressedReadBuffer.h @@ -61,14 +61,14 @@ public: void setReadUntilPosition(size_t position) override { - if (file_in) - file_in->setReadUntilPosition(position); + initInput(); + file_in->setReadUntilPosition(position); } void setReadUntilEnd() override { - if (file_in) - file_in->setReadUntilEnd(); + initInput(); + file_in->setReadUntilEnd(); } }; diff --git a/src/Compression/CompressedReadBufferFromFile.cpp b/src/Compression/CompressedReadBufferFromFile.cpp index 1a70b27e9f4..cf08d68a7aa 100644 --- a/src/Compression/CompressedReadBufferFromFile.cpp +++ b/src/Compression/CompressedReadBufferFromFile.cpp @@ -80,7 +80,7 @@ void CompressedReadBufferFromFile::seek(size_t offset_in_compressed_file, size_t /// We will discard our working_buffer, but have to account rest bytes bytes += offset(); /// No data, everything discarded - pos = working_buffer.end(); + resetWorkingBuffer(); size_compressed = 0; /// Remember required offset in decompressed block which will be set in /// the next ReadBuffer::next() call @@ -113,7 +113,6 @@ size_t CompressedReadBufferFromFile::readBig(char * to, size_t n) /// need to skip some bytes in decompressed data (seek happened before readBig call). if (nextimpl_working_buffer_offset == 0 && size_decompressed + additional_size_at_the_end_of_buffer <= n - bytes_read) { - decompressTo(to + bytes_read, size_decompressed, size_compressed_without_checksum); bytes_read += size_decompressed; bytes += size_decompressed; diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index 74e093284a8..e82c1dacdd7 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -54,11 +54,6 @@ ChangelogFileDescription getChangelogFileDescription(const std::string & path_st return result; } -LogEntryPtr makeClone(const LogEntryPtr & entry) -{ - return cs_new(entry->get_term(), nuraft::buffer::clone(entry->get_buf()), entry->get_val_type()); -} - Checksum computeRecordChecksum(const ChangelogRecord & record) { SipHash hash; @@ -519,7 +514,7 @@ void Changelog::appendEntry(uint64_t index, const LogEntryPtr & log_entry) rotate(index); current_writer->appendRecord(buildRecord(index, log_entry)); - logs[index] = makeClone(log_entry); + logs[index] = log_entry; max_log_id = index; } diff --git a/src/Coordination/KeeperServer.cpp b/src/Coordination/KeeperServer.cpp index 25d57e64e0a..558b28f9d46 100644 --- a/src/Coordination/KeeperServer.cpp +++ b/src/Coordination/KeeperServer.cpp @@ -286,10 +286,7 @@ RaftAppendResult KeeperServer::putRequestBatch(const KeeperStorage::RequestsForS for (const auto & [session_id, request] : requests_for_sessions) entries.push_back(getZooKeeperLogEntry(session_id, request)); - { - std::lock_guard lock(append_entries_mutex); - return raft_instance->append_entries(entries); - } + return raft_instance->append_entries(entries); } bool KeeperServer::isLeader() const diff --git a/src/Coordination/KeeperServer.h b/src/Coordination/KeeperServer.h index 1fb02bb0987..4ed88ceb855 100644 --- a/src/Coordination/KeeperServer.h +++ b/src/Coordination/KeeperServer.h @@ -28,8 +28,6 @@ private: nuraft::ptr asio_service; nuraft::ptr asio_listener; - std::mutex append_entries_mutex; - std::mutex initialized_mutex; std::atomic initialized_flag = false; std::condition_variable initialized_cv; diff --git a/src/Coordination/LoggerWrapper.h b/src/Coordination/LoggerWrapper.h index 002fa870241..a2493763633 100644 --- a/src/Coordination/LoggerWrapper.h +++ b/src/Coordination/LoggerWrapper.h @@ -39,7 +39,7 @@ public: const std::string & msg) override { LogsLevel db_level = static_cast(level_); - LOG_IMPL(log, db_level, LEVELS.at(db_level), msg); + LOG_IMPL(log, db_level, LEVELS.at(db_level), fmt::runtime(msg)); } void set_level(int level_) override diff --git a/src/Coordination/SnapshotableHashTable.h b/src/Coordination/SnapshotableHashTable.h index b1d72578530..c3a6d7a8eff 100644 --- a/src/Coordination/SnapshotableHashTable.h +++ b/src/Coordination/SnapshotableHashTable.h @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -36,6 +37,8 @@ private: /// Allows to avoid additional copies in updateValue function size_t snapshot_up_to_size = 0; ArenaWithFreeLists arena; + /// Collect invalid iterators to avoid traversing the whole list + std::vector snapshot_invalid_iters; uint64_t approximate_data_size{0}; @@ -113,17 +116,6 @@ private: } } - StringRef copyStringInArena(const std::string & value_to_copy) - { - size_t value_to_copy_size = value_to_copy.size(); - char * place_for_key = arena.alloc(value_to_copy_size); - memcpy(reinterpret_cast(place_for_key), reinterpret_cast(value_to_copy.data()), value_to_copy_size); - StringRef updated_value{place_for_key, value_to_copy_size}; - - return updated_value; - } - - public: using iterator = typename List::iterator; @@ -137,7 +129,7 @@ public: if (!it) { - ListElem elem{copyStringInArena(key), value, true}; + ListElem elem{copyStringInArena(arena, key), value, true}; auto itr = list.insert(list.end(), elem); bool inserted; map.emplace(itr->key, it, inserted, hash_value); @@ -159,7 +151,7 @@ public: if (it == map.end()) { - ListElem elem{copyStringInArena(key), value, true}; + ListElem elem{copyStringInArena(arena, key), value, true}; auto itr = list.insert(list.end(), elem); bool inserted; map.emplace(itr->key, it, inserted, hash_value); @@ -175,6 +167,7 @@ public: list_itr->active_in_map = false; auto new_list_itr = list.insert(list.end(), elem); it->getMapped() = new_list_itr; + snapshot_invalid_iters.push_back(list_itr); } else { @@ -195,6 +188,7 @@ public: if (snapshot_mode) { list_itr->active_in_map = false; + snapshot_invalid_iters.push_back(list_itr); list_itr->free_key = true; map.erase(it->getKey()); } @@ -235,6 +229,7 @@ public: { auto elem_copy = *(list_itr); list_itr->active_in_map = false; + snapshot_invalid_iters.push_back(list_itr); updater(elem_copy.value); auto itr = list.insert(list.end(), elem_copy); it->getMapped() = itr; @@ -274,23 +269,15 @@ public: void clearOutdatedNodes() { - auto start = list.begin(); - auto end = list.end(); - for (auto itr = start; itr != end;) + for (auto & itr: snapshot_invalid_iters) { - if (!itr->active_in_map) - { - updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size, itr->value.sizeInBytes(), 0); - if (itr->free_key) - arena.free(const_cast(itr->key.data), itr->key.size); - itr = list.erase(itr); - } - else - { - assert(!itr->free_key); - itr++; - } + assert(!itr->active_in_map); + updateDataSize(CLEAR_OUTDATED_NODES, itr->key.size, itr->value.sizeInBytes(), 0); + if (itr->free_key) + arena.free(const_cast(itr->key.data), itr->key.size); + list.erase(itr); } + snapshot_invalid_iters.clear(); } void clear() @@ -310,7 +297,6 @@ public: void disableSnapshotMode() { - snapshot_mode = false; snapshot_up_to_size = 0; } diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 9a42f752db2..18c43d8c45f 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -5,7 +5,6 @@ #include #include #include -#include namespace DB @@ -246,7 +245,6 @@ void BackgroundSchedulePool::threadFunction() setThreadName(thread_name.c_str()); attachToThreadGroup(); - SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); while (!shutdown) { @@ -273,7 +271,6 @@ void BackgroundSchedulePool::delayExecutionThreadFunction() setThreadName((thread_name + "/D").c_str()); attachToThreadGroup(); - SCOPE_EXIT({ CurrentThread::detachQueryIfNotDetached(); }); while (!shutdown) { diff --git a/src/Core/Block.cpp b/src/Core/Block.cpp index 07db1a1fafa..26c883b308d 100644 --- a/src/Core/Block.cpp +++ b/src/Core/Block.cpp @@ -12,6 +12,7 @@ #include #include +#include namespace DB @@ -538,7 +539,7 @@ Block Block::sortColumns() const for (auto it = index_by_name.begin(); it != index_by_name.end(); ++it) sorted_index_by_name[i++] = it; } - std::sort(sorted_index_by_name.begin(), sorted_index_by_name.end(), [](const auto & lhs, const auto & rhs) + ::sort(sorted_index_by_name.begin(), sorted_index_by_name.end(), [](const auto & lhs, const auto & rhs) { return lhs->first < rhs->first; }); @@ -754,4 +755,30 @@ void materializeBlockInplace(Block & block) block.getByPosition(i).column = recursiveRemoveSparse(block.getByPosition(i).column->convertToFullColumnIfConst()); } +Block concatenateBlocks(const std::vector & blocks) +{ + if (blocks.empty()) + return {}; + + size_t num_rows = 0; + for (const auto & block : blocks) + num_rows += block.rows(); + + Block out = blocks[0].cloneEmpty(); + MutableColumns columns = out.mutateColumns(); + + for (size_t i = 0; i < columns.size(); ++i) + { + columns[i]->reserve(num_rows); + for (const auto & block : blocks) + { + const auto & tmp_column = *block.getByPosition(i).column; + columns[i]->insertRangeFrom(tmp_column, 0, block.rows()); + } + } + + out.setColumns(std::move(columns)); + return out; +} + } diff --git a/src/Core/Block.h b/src/Core/Block.h index efa5ce7c326..2624b57880c 100644 --- a/src/Core/Block.h +++ b/src/Core/Block.h @@ -203,4 +203,6 @@ ColumnPtr getColumnFromBlock(const Block & block, const NameAndTypePair & column Block materializeBlock(const Block & block); void materializeBlockInplace(Block & block); +Block concatenateBlocks(const std::vector & blocks); + } diff --git a/src/Core/NamesAndTypes.cpp b/src/Core/NamesAndTypes.cpp index b9098d3308d..be947623a96 100644 --- a/src/Core/NamesAndTypes.cpp +++ b/src/Core/NamesAndTypes.cpp @@ -1,4 +1,6 @@ #include + +#include #include #include #include @@ -113,7 +115,7 @@ bool NamesAndTypesList::isSubsetOf(const NamesAndTypesList & rhs) const { NamesAndTypes vector(rhs.begin(), rhs.end()); vector.insert(vector.end(), begin(), end()); - std::sort(vector.begin(), vector.end()); + ::sort(vector.begin(), vector.end()); return std::unique(vector.begin(), vector.end()) == vector.begin() + rhs.size(); } @@ -121,16 +123,16 @@ size_t NamesAndTypesList::sizeOfDifference(const NamesAndTypesList & rhs) const { NamesAndTypes vector(rhs.begin(), rhs.end()); vector.insert(vector.end(), begin(), end()); - std::sort(vector.begin(), vector.end()); + ::sort(vector.begin(), vector.end()); return (std::unique(vector.begin(), vector.end()) - vector.begin()) * 2 - size() - rhs.size(); } void NamesAndTypesList::getDifference(const NamesAndTypesList & rhs, NamesAndTypesList & deleted, NamesAndTypesList & added) const { NamesAndTypes lhs_vector(begin(), end()); - std::sort(lhs_vector.begin(), lhs_vector.end()); + ::sort(lhs_vector.begin(), lhs_vector.end()); NamesAndTypes rhs_vector(rhs.begin(), rhs.end()); - std::sort(rhs_vector.begin(), rhs_vector.end()); + ::sort(rhs_vector.begin(), rhs_vector.end()); std::set_difference(lhs_vector.begin(), lhs_vector.end(), rhs_vector.begin(), rhs_vector.end(), std::back_inserter(deleted)); diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 8daf39d9928..87d7eee0daa 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -85,16 +85,18 @@ void Settings::addProgramOptions(boost::program_options::options_description & o { for (const auto & field : all()) { - const std::string_view name = field.getName(); - auto on_program_option - = boost::function1([this, name](const std::string & value) { set(name, value); }); - options.add(boost::shared_ptr(new boost::program_options::option_description( - name.data(), - boost::program_options::value()->composing()->notifier(on_program_option), - field.getDescription()))); + addProgramOption(options, field); } } +void Settings::addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field) +{ + const std::string_view name = field.getName(); + auto on_program_option = boost::function1([this, name](const std::string & value) { set(name, value); }); + options.add(boost::shared_ptr(new boost::program_options::option_description( + name.data(), boost::program_options::value()->composing()->notifier(on_program_option), field.getDescription()))); +} + void Settings::checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path) { if (config.getBool("skip_check_for_incorrect_settings", false)) diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 0aae455d058..25c3ddbe582 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -70,7 +70,9 @@ class IColumn; M(UInt64, idle_connection_timeout, 3600, "Close idle TCP connections after specified number of seconds.", 0) \ M(UInt64, distributed_connections_pool_size, 1024, "Maximum number of connections with one remote server in the pool.", 0) \ M(UInt64, connections_with_failover_max_tries, DBMS_CONNECTION_POOL_WITH_FAILOVER_DEFAULT_MAX_TRIES, "The maximum number of attempts to connect to replicas.", 0) \ - M(UInt64, s3_min_upload_part_size, 32*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \ + M(UInt64, s3_min_upload_part_size, 16*1024*1024, "The minimum size of part to upload during multipart upload to S3.", 0) \ + M(UInt64, s3_upload_part_size_multiply_factor, 2, "Multiply s3_min_upload_part_size by this factor each time s3_multiply_parts_count_threshold parts were uploaded from a single write to S3.", 0) \ + M(UInt64, s3_upload_part_size_multiply_parts_count_threshold, 1000, "Each time this number of parts was uploaded to S3 s3_min_upload_part_size multiplied by s3_upload_part_size_multiply_factor.", 0) \ M(UInt64, s3_max_single_part_upload_size, 32*1024*1024, "The maximum size of object to upload using singlepart upload to S3.", 0) \ M(UInt64, s3_max_single_read_retries, 4, "The maximum number of retries during single S3 read.", 0) \ M(UInt64, s3_max_redirects, 10, "Max number of S3 redirects hops allowed.", 0) \ @@ -170,6 +172,7 @@ class IColumn; M(Bool, force_index_by_date, false, "Throw an exception if there is a partition key in a table, and it is not used.", 0) \ M(Bool, force_primary_key, false, "Throw an exception if there is primary key in a table, and it is not used.", 0) \ M(Bool, use_skip_indexes, true, "Use data skipping indexes during query execution.", 0) \ + M(Bool, use_skip_indexes_if_final, false, "If query has FINAL, then skipping data based on indexes may produce incorrect result, hence disabled by default.", 0) \ M(String, force_data_skipping_indices, "", "Comma separated list of strings or literals with the name of the data skipping indices that should be used during query execution, otherwise an exception will be thrown.", 0) \ \ M(Float, max_streams_to_max_threads_ratio, 1, "Allows you to use more sources than the number of threads - to more evenly distribute work across threads. It is assumed that this is a temporary solution, since it will be possible in the future to make the number of sources equal to the number of threads, but for each source to dynamically select available work for itself.", 0) \ @@ -261,6 +264,7 @@ class IColumn; M(UInt64, http_max_fields, 1000000, "Maximum number of fields in HTTP header", 0) \ M(UInt64, http_max_field_name_size, 1048576, "Maximum length of field name in HTTP header", 0) \ M(UInt64, http_max_field_value_size, 1048576, "Maximum length of field value in HTTP header", 0) \ + M(Bool, http_skip_not_found_url_for_globs, true, "Skip url's for globs with HTTP_NOT_FOUND error", 0) \ M(Bool, optimize_throw_if_noop, false, "If setting is enabled and OPTIMIZE query didn't actually assign a merge then an explanatory exception is thrown", 0) \ M(Bool, use_index_for_in_with_subqueries, true, "Try using an index if there is a subquery or a table expression on the right side of the IN operator.", 0) \ M(Bool, joined_subquery_requires_alias, true, "Force joined subqueries and table functions to have aliases for correct name qualification.", 0) \ @@ -354,11 +358,15 @@ class IColumn; M(OverflowMode, distinct_overflow_mode, OverflowMode::THROW, "What to do when the limit is exceeded.", 0) \ \ M(UInt64, max_memory_usage, 0, "Maximum memory usage for processing of single query. Zero means unlimited.", 0) \ + M(UInt64, max_guaranteed_memory_usage, 0, "Maximum guaranteed memory usage for processing of single query. It represents soft limit. Zero means unlimited.", 0) \ M(UInt64, max_memory_usage_for_user, 0, "Maximum memory usage for processing all concurrently running queries for the user. Zero means unlimited.", 0) \ + M(UInt64, max_guaranteed_memory_usage_for_user, 0, "Maximum guaranteed memory usage for processing all concurrently running queries for the user. It represents soft limit. Zero means unlimited.", 0) \ M(UInt64, max_untracked_memory, (4 * 1024 * 1024), "Small allocations and deallocations are grouped in thread local variable and tracked or profiled only when amount (in absolute value) becomes larger than specified value. If the value is higher than 'memory_profiler_step' it will be effectively lowered to 'memory_profiler_step'.", 0) \ M(UInt64, memory_profiler_step, (4 * 1024 * 1024), "Whenever query memory usage becomes larger than every next step in number of bytes the memory profiler will collect the allocating stack trace. Zero means disabled memory profiler. Values lower than a few megabytes will slow down query processing.", 0) \ M(Float, memory_profiler_sample_probability, 0., "Collect random allocations and deallocations and write them into system.trace_log with 'MemorySample' trace_type. The probability is for every alloc/free regardless to the size of the allocation. Note that sampling happens only when the amount of untracked memory exceeds 'max_untracked_memory'. You may want to set 'max_untracked_memory' to 0 for extra fine grained sampling.", 0) \ \ + M(UInt64, memory_usage_overcommit_max_wait_microseconds, 0, "Maximum time thread will wait for memory to be freed in the case of memory overcommit. If timeout is reached and memory is not freed, exception is thrown", 0) \ + \ M(UInt64, max_network_bandwidth, 0, "The maximum speed of data exchange over the network in bytes per second for a query. Zero means unlimited.", 0) \ M(UInt64, max_network_bytes, 0, "The maximum number of bytes (compressed) to receive or transmit over the network for execution of the query.", 0) \ M(UInt64, max_network_bandwidth_for_user, 0, "The maximum speed of data exchange over the network in bytes per second for all concurrently running user queries. Zero means unlimited.", 0)\ @@ -425,6 +433,7 @@ class IColumn; M(UInt64, min_free_disk_space_for_temporary_data, 0, "The minimum disk space to keep while writing temporary data used in external sorting and aggregation.", 0) \ \ M(DefaultDatabaseEngine, default_database_engine, DefaultDatabaseEngine::Atomic, "Default database engine.", 0) \ + M(DefaultTableEngine, default_table_engine, DefaultTableEngine::None, "Default table engine used when ENGINE is not set in CREATE statement.",0) \ M(Bool, show_table_uuid_in_table_create_query_if_not_nil, false, "For tables in databases with Engine=Atomic show UUID of the table in its CREATE query.", 0) \ M(Bool, database_atomic_wait_for_drop_and_detach_synchronously, false, "When executing DROP or DETACH TABLE in Atomic database, wait for table data to be finally dropped or detached.", 0) \ M(Bool, enable_scalar_subquery_optimization, true, "If it is set to true, prevent scalar subqueries from (de)serializing large scalar values and possibly avoid running the same subquery more than once.", 0) \ @@ -478,7 +487,6 @@ class IColumn; M(Bool, asterisk_include_alias_columns, false, "Include ALIAS columns for wildcard query", 0) \ M(Bool, optimize_skip_merged_partitions, false, "Skip partitions with one part with level > 0 in optimize final", 0) \ M(Bool, optimize_on_insert, true, "Do the same transformation for inserted block of data as if merge was done on this block.", 0) \ - M(Bool, allow_experimental_projection_optimization, false, "Enable projection optimization when processing SELECT queries", 0) \ M(Bool, force_optimize_projection, false, "If projection optimization is enabled, SELECT queries need to use projection", 0) \ M(Bool, async_socket_for_remote, true, "Asynchronously read from socket executing remote query", 0) \ M(Bool, insert_null_as_default, true, "Insert DEFAULT values instead of NULL in INSERT SELECT (UNION ALL)", 0) \ @@ -543,7 +551,7 @@ class IColumn; M(Int64, remote_fs_read_max_backoff_ms, 10000, "Max wait time when trying to read data for remote disk", 0) \ M(Int64, remote_fs_read_backoff_max_tries, 5, "Max attempts to read with backoff", 0) \ \ - M(UInt64, http_max_tries, 1, "Max attempts to read via http.", 0) \ + M(UInt64, http_max_tries, 10, "Max attempts to read via http.", 0) \ M(UInt64, http_retry_initial_backoff_ms, 100, "Min milliseconds for backoff, when retrying read via http", 0) \ M(UInt64, http_retry_max_backoff_ms, 10000, "Max milliseconds for backoff, when retrying read via http", 0) \ \ @@ -554,7 +562,7 @@ class IColumn; /** Experimental functions */ \ M(Bool, allow_experimental_funnel_functions, false, "Enable experimental functions for funnel analysis.", 0) \ M(Bool, allow_experimental_nlp_functions, false, "Enable experimental functions for natural language processing.", 0) \ - + M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \ // End of COMMON_SETTINGS // Please add settings related to formats into the FORMAT_FACTORY_SETTINGS and move obsolete settings to OBSOLETE_SETTINGS. @@ -578,6 +586,7 @@ class IColumn; MAKE_OBSOLETE(M, UInt64, merge_tree_clear_old_parts_interval_seconds, 1) \ MAKE_OBSOLETE(M, UInt64, partial_merge_join_optimizations, 0) \ MAKE_OBSOLETE(M, MaxThreads, max_alter_threads, 0) \ + MAKE_OBSOLETE(M, Bool, allow_experimental_projection_optimization, true) \ /** The section above is for obsolete settings. Do not add anything there. */ @@ -609,6 +618,7 @@ class IColumn; M(Char, input_format_hive_text_collection_items_delimiter, '\x02', "Delimiter between collection(array or map) items in Hive Text File", 0) \ M(Char, input_format_hive_text_map_keys_delimiter, '\x03', "Delimiter between a pair of map key/values in Hive Text File", 0) \ M(UInt64, input_format_msgpack_number_of_columns, 0, "The number of columns in inserted MsgPack data. Used for automatic schema inference from data.", 0) \ + M(MsgPackUUIDRepresentation, output_format_msgpack_uuid_representation, FormatSettings::MsgPackUUIDRepresentation::EXT, "The way how to output UUID in MsgPack format.", 0) \ M(UInt64, input_format_max_rows_to_read_for_schema_inference, 100, "The maximum rows of data to read for automatic schema inference", 0) \ \ M(DateTimeInputFormat, date_time_input_format, FormatSettings::DateTimeInputFormat::Basic, "Method to read DateTime from text input formats. Possible values: 'basic' and 'best_effort'.", 0) \ @@ -716,6 +726,8 @@ struct Settings : public BaseSettings, public IHints<2, Settings static void checkNoSettingNamesAtTopLevel(const Poco::Util::AbstractConfiguration & config, const String & config_path); std::vector getAllRegisteredNames() const override; + + void addProgramOption(boost::program_options::options_description & options, const SettingFieldRef & field); }; /* diff --git a/src/Core/SettingsEnums.cpp b/src/Core/SettingsEnums.cpp index b62575c9730..17d24946cd8 100644 --- a/src/Core/SettingsEnums.cpp +++ b/src/Core/SettingsEnums.cpp @@ -93,6 +93,16 @@ IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultDatabaseEngine, ErrorCodes::BAD_ARGUME {{"Ordinary", DefaultDatabaseEngine::Ordinary}, {"Atomic", DefaultDatabaseEngine::Atomic}}) +IMPLEMENT_SETTING_ENUM_WITH_RENAME(DefaultTableEngine, ErrorCodes::BAD_ARGUMENTS, + {{"None", DefaultTableEngine::None}, + {"Log", DefaultTableEngine::Log}, + {"StripeLog", DefaultTableEngine::StripeLog}, + {"MergeTree", DefaultTableEngine::MergeTree}, + {"ReplacingMergeTree", DefaultTableEngine::ReplacingMergeTree}, + {"ReplicatedMergeTree", DefaultTableEngine::ReplicatedMergeTree}, + {"ReplicatedReplacingMergeTree", DefaultTableEngine::ReplicatedReplacingMergeTree}, + {"Memory", DefaultTableEngine::Memory}}) + IMPLEMENT_SETTING_MULTI_ENUM(MySQLDataTypesSupport, ErrorCodes::UNKNOWN_MYSQL_DATATYPES_SUPPORT_LEVEL, {{"decimal", MySQLDataTypesSupport::DECIMAL}, {"datetime64", MySQLDataTypesSupport::DATETIME64}}) @@ -130,4 +140,10 @@ IMPLEMENT_SETTING_ENUM(EscapingRule, ErrorCodes::BAD_ARGUMENTS, {"JSON", FormatSettings::EscapingRule::JSON}, {"XML", FormatSettings::EscapingRule::XML}, {"Raw", FormatSettings::EscapingRule::Raw}}) + +IMPLEMENT_SETTING_ENUM(MsgPackUUIDRepresentation , ErrorCodes::BAD_ARGUMENTS, + {{"bin", FormatSettings::MsgPackUUIDRepresentation::BIN}, + {"str", FormatSettings::MsgPackUUIDRepresentation::STR}, + {"ext", FormatSettings::MsgPackUUIDRepresentation::EXT}}) + } diff --git a/src/Core/SettingsEnums.h b/src/Core/SettingsEnums.h index 106589f5d24..27994529a0b 100644 --- a/src/Core/SettingsEnums.h +++ b/src/Core/SettingsEnums.h @@ -120,6 +120,19 @@ enum class DefaultDatabaseEngine DECLARE_SETTING_ENUM(DefaultDatabaseEngine) +enum class DefaultTableEngine +{ + None = 0, /// Disable. Need to use ENGINE = + Log, + StripeLog, + MergeTree, + ReplacingMergeTree, + ReplicatedMergeTree, + ReplicatedReplacingMergeTree, + Memory, +}; + +DECLARE_SETTING_ENUM(DefaultTableEngine) enum class MySQLDataTypesSupport { @@ -172,4 +185,6 @@ DECLARE_SETTING_ENUM_WITH_RENAME(EnumComparingMode, FormatSettings::EnumComparin DECLARE_SETTING_ENUM_WITH_RENAME(EscapingRule, FormatSettings::EscapingRule) +DECLARE_SETTING_ENUM_WITH_RENAME(MsgPackUUIDRepresentation, FormatSettings::MsgPackUUIDRepresentation) + } diff --git a/src/DataTypes/DataTypeLowCardinality.h b/src/DataTypes/DataTypeLowCardinality.h index 38b2109eec6..57f67ddad7a 100644 --- a/src/DataTypes/DataTypeLowCardinality.h +++ b/src/DataTypes/DataTypeLowCardinality.h @@ -13,7 +13,7 @@ private: DataTypePtr dictionary_type; public: - DataTypeLowCardinality(DataTypePtr dictionary_type_); + explicit DataTypeLowCardinality(DataTypePtr dictionary_type_); const DataTypePtr & getDictionaryType() const { return dictionary_type; } diff --git a/src/DataTypes/DataTypeTuple.cpp b/src/DataTypes/DataTypeTuple.cpp index ad6d4e2943b..a5e9868cf89 100644 --- a/src/DataTypes/DataTypeTuple.cpp +++ b/src/DataTypes/DataTypeTuple.cpp @@ -32,6 +32,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int SIZES_OF_COLUMNS_IN_TUPLE_DOESNT_MATCH; extern const int ILLEGAL_INDEX; + extern const int LOGICAL_ERROR; } @@ -156,8 +157,19 @@ MutableColumnPtr DataTypeTuple::createColumn() const MutableColumnPtr DataTypeTuple::createColumn(const ISerialization & serialization) const { - const auto & element_serializations = - assert_cast(serialization).getElementsSerializations(); + /// If we read subcolumn of nested Tuple, it may be wrapped to SerializationNamed + /// several times to allow to reconstruct the substream path name. + /// Here we don't need substream path name, so we drop first several wrapper serializations. + + const auto * current_serialization = &serialization; + while (const auto * serialization_named = typeid_cast(current_serialization)) + current_serialization = serialization_named->getNested().get(); + + const auto * serialization_tuple = typeid_cast(current_serialization); + if (!serialization_tuple) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected serialization to create column of type Tuple"); + + const auto & element_serializations = serialization_tuple->getElementsSerializations(); size_t size = elems.size(); assert(element_serializations.size() == size); diff --git a/src/DataTypes/EnumValues.cpp b/src/DataTypes/EnumValues.cpp index ab5ea0ca249..ffa8f55a974 100644 --- a/src/DataTypes/EnumValues.cpp +++ b/src/DataTypes/EnumValues.cpp @@ -1,5 +1,7 @@ #include #include +#include + namespace DB { @@ -18,7 +20,7 @@ EnumValues::EnumValues(const Values & values_) if (values.empty()) throw Exception{"DataTypeEnum enumeration cannot be empty", ErrorCodes::EMPTY_DATA_PASSED}; - std::sort(std::begin(values), std::end(values), [] (auto & left, auto & right) + ::sort(std::begin(values), std::end(values), [] (auto & left, auto & right) { return left.second < right.second; }); diff --git a/src/DataTypes/Serializations/ISerialization.cpp b/src/DataTypes/Serializations/ISerialization.cpp index 5cdc037d5cb..7df4a956c1a 100644 --- a/src/DataTypes/Serializations/ISerialization.cpp +++ b/src/DataTypes/Serializations/ISerialization.cpp @@ -167,8 +167,10 @@ String getNameForSubstreamPath( /// Because nested data may be represented not by Array of Tuple, /// but by separate Array columns with names in a form of a.b, /// and name is encoded as a whole. - stream_name += (escape_tuple_delimiter && it->escape_tuple_delimiter ? - escapeForFileName(".") : ".") + escapeForFileName(it->tuple_element_name); + if (escape_tuple_delimiter && it->escape_tuple_delimiter) + stream_name += escapeForFileName("." + it->tuple_element_name); + else + stream_name += "." + it->tuple_element_name; } } diff --git a/src/DataTypes/Serializations/SerializationArray.cpp b/src/DataTypes/Serializations/SerializationArray.cpp index e3b535a2a11..30ee5e98b74 100644 --- a/src/DataTypes/Serializations/SerializationArray.cpp +++ b/src/DataTypes/Serializations/SerializationArray.cpp @@ -37,10 +37,11 @@ void SerializationArray::deserializeBinary(Field & field, ReadBuffer & istr) con { size_t size; readVarUInt(size, istr); - field = Array(size); + field = Array(); Array & arr = get(field); + arr.reserve(size); for (size_t i = 0; i < size; ++i) - nested->deserializeBinary(arr[i], istr); + nested->deserializeBinary(arr.emplace_back(), istr); } diff --git a/src/DataTypes/Serializations/SerializationMap.cpp b/src/DataTypes/Serializations/SerializationMap.cpp index 3f17061a744..24d06d8f3b2 100644 --- a/src/DataTypes/Serializations/SerializationMap.cpp +++ b/src/DataTypes/Serializations/SerializationMap.cpp @@ -53,13 +53,15 @@ void SerializationMap::deserializeBinary(Field & field, ReadBuffer & istr) const { size_t size; readVarUInt(size, istr); - field = Map(size); - for (auto & elem : field.get()) + field = Map(); + Map & map = field.get(); + map.reserve(size); + for (size_t i = 0; i < size; ++i) { Tuple tuple(2); key->deserializeBinary(tuple[0], istr); value->deserializeBinary(tuple[1], istr); - elem = std::move(tuple); + map.push_back(std::move(tuple)); } } diff --git a/src/DataTypes/Serializations/SerializationNamed.h b/src/DataTypes/Serializations/SerializationNamed.h index 91db0cf67f4..343b96c16e3 100644 --- a/src/DataTypes/Serializations/SerializationNamed.h +++ b/src/DataTypes/Serializations/SerializationNamed.h @@ -5,6 +5,11 @@ namespace DB { +/// Serialization wrapper that acts like nested serialization, +/// but adds a passed name to the substream path like the +/// read column was the tuple element with this name. +/// It's used while reading subcolumns of complex types. +/// In particular while reading components of named tuples. class SerializationNamed final : public SerializationWrapper { private: diff --git a/src/DataTypes/Serializations/SerializationTuple.cpp b/src/DataTypes/Serializations/SerializationTuple.cpp index cd5a6b65a3c..8dc15fc9841 100644 --- a/src/DataTypes/Serializations/SerializationTuple.cpp +++ b/src/DataTypes/Serializations/SerializationTuple.cpp @@ -1,4 +1,3 @@ -#include #include #include #include @@ -44,11 +43,11 @@ void SerializationTuple::deserializeBinary(Field & field, ReadBuffer & istr) con { const size_t size = elems.size(); - Tuple tuple(size); - for (const auto i : collections::range(0, size)) - elems[i]->deserializeBinary(tuple[i], istr); - - field = tuple; + field = Tuple(); + Tuple & tuple = get(field); + tuple.reserve(size); + for (size_t i = 0; i < size; ++i) + elems[i]->deserializeBinary(tuple.emplace_back(), istr); } void SerializationTuple::serializeBinary(const IColumn & column, size_t row_num, WriteBuffer & ostr) const @@ -73,7 +72,7 @@ static void addElementSafe(size_t num_elems, IColumn & column, F && impl) // Check that all columns now have the same size. size_t new_size = column.size(); - for (auto i : collections::range(1, num_elems)) + for (size_t i = 1; i < num_elems; ++i) { const auto & element_column = extractElementColumn(column, i); if (element_column.size() != new_size) @@ -87,7 +86,7 @@ static void addElementSafe(size_t num_elems, IColumn & column, F && impl) } catch (...) { - for (const auto & i : collections::range(0, num_elems)) + for (size_t i = 0; i < num_elems; ++i) { auto & element_column = extractElementColumn(column, i); if (element_column.size() > old_size) @@ -102,7 +101,7 @@ void SerializationTuple::deserializeBinary(IColumn & column, ReadBuffer & istr) { addElementSafe(elems.size(), column, [&] { - for (const auto & i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) elems[i]->deserializeBinary(extractElementColumn(column, i), istr); }); } @@ -110,7 +109,7 @@ void SerializationTuple::deserializeBinary(IColumn & column, ReadBuffer & istr) void SerializationTuple::serializeText(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { writeChar('(', ostr); - for (const auto i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) { if (i != 0) writeChar(',', ostr); @@ -126,7 +125,7 @@ void SerializationTuple::deserializeText(IColumn & column, ReadBuffer & istr, co addElementSafe(elems.size(), column, [&] { - for (const auto i : collections::range(0, size)) + for (size_t i = 0; i < size; ++i) { skipWhitespaceIfAny(istr); if (i != 0) @@ -158,7 +157,7 @@ void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_nu && have_explicit_names) { writeChar('{', ostr); - for (const auto i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) { if (i != 0) { @@ -173,7 +172,7 @@ void SerializationTuple::serializeTextJSON(const IColumn & column, size_t row_nu else { writeChar('[', ostr); - for (const auto i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) { if (i != 0) writeChar(',', ostr); @@ -195,7 +194,7 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr addElementSafe(elems.size(), column, [&] { // Require all elements but in arbitrary order. - for (auto i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) { if (i > 0) { @@ -226,7 +225,7 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr addElementSafe(elems.size(), column, [&] { - for (const auto i : collections::range(0, size)) + for (size_t i = 0; i < size; ++i) { skipWhitespaceIfAny(istr); if (i != 0) @@ -246,7 +245,7 @@ void SerializationTuple::deserializeTextJSON(IColumn & column, ReadBuffer & istr void SerializationTuple::serializeTextXML(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { writeCString("", ostr); - for (const auto i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) { writeCString("", ostr); elems[i]->serializeTextXML(extractElementColumn(column, i), row_num, ostr, settings); @@ -257,7 +256,7 @@ void SerializationTuple::serializeTextXML(const IColumn & column, size_t row_num void SerializationTuple::serializeTextCSV(const IColumn & column, size_t row_num, WriteBuffer & ostr, const FormatSettings & settings) const { - for (const auto i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) { if (i != 0) writeChar(settings.csv.tuple_delimiter, ostr); @@ -270,7 +269,7 @@ void SerializationTuple::deserializeTextCSV(IColumn & column, ReadBuffer & istr, addElementSafe(elems.size(), column, [&] { const size_t size = elems.size(); - for (const auto i : collections::range(0, size)) + for (size_t i = 0; i < size; ++i) { if (i != 0) { @@ -362,7 +361,7 @@ void SerializationTuple::serializeBinaryBulkWithMultipleStreams( { auto * tuple_state = checkAndGetState(state); - for (const auto i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) { const auto & element_col = extractElementColumn(column, i); elems[i]->serializeBinaryBulkWithMultipleStreams(element_col, offset, limit, settings, tuple_state->states[i]); @@ -382,7 +381,7 @@ void SerializationTuple::deserializeBinaryBulkWithMultipleStreams( auto & column_tuple = assert_cast(*mutable_column); settings.avg_value_size_hint = 0; - for (const auto i : collections::range(0, elems.size())) + for (size_t i = 0; i < elems.size(); ++i) elems[i]->deserializeBinaryBulkWithMultipleStreams(column_tuple.getColumnPtr(i), limit, settings, tuple_state->states[i], cache); } diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp index cb0c1cdae95..721bf79199b 100644 --- a/src/Databases/DatabaseAtomic.cpp +++ b/src/Databases/DatabaseAtomic.cpp @@ -80,7 +80,7 @@ void DatabaseAtomic::drop(ContextPtr) } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(true)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true))); } fs::remove_all(getMetadataPath()); } @@ -469,7 +469,7 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String & } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(true)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true))); } } @@ -482,7 +482,7 @@ void DatabaseAtomic::tryRemoveSymlink(const String & table_name) } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(true)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true))); } } @@ -527,7 +527,7 @@ void DatabaseAtomic::renameDatabase(ContextPtr query_context, const String & new } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(true)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true))); } auto new_name_escaped = escapeForFileName(new_name); diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp index 1ff84b53eee..7f8f1b917d7 100644 --- a/src/Databases/DatabaseLazy.cpp +++ b/src/Databases/DatabaseLazy.cpp @@ -12,6 +12,7 @@ #include #include +#include #include #include @@ -151,7 +152,7 @@ DatabaseTablesIteratorPtr DatabaseLazy::getTablesIterator(ContextPtr, const Filt if (!filter_by_table_name || filter_by_table_name(table_name)) filtered_tables.push_back(table_name); } - std::sort(filtered_tables.begin(), filtered_tables.end()); + ::sort(filtered_tables.begin(), filtered_tables.end()); return std::make_unique(*this, std::move(filtered_tables)); } diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp index 3309d25b1c2..a92c19f67c0 100644 --- a/src/Databases/DatabaseMemory.cpp +++ b/src/Databases/DatabaseMemory.cpp @@ -78,7 +78,9 @@ ASTPtr DatabaseMemory::getCreateDatabaseQuery() const auto create_query = std::make_shared(); create_query->setDatabase(getDatabaseName()); create_query->set(create_query->storage, std::make_shared()); - create_query->storage->set(create_query->storage->engine, makeASTFunction(getEngineName())); + auto engine = makeASTFunction(getEngineName()); + engine->no_empty_args = true; + create_query->storage->set(create_query->storage->engine, engine); if (const auto comment_value = getDatabaseComment(); !comment_value.empty()) create_query->set(create_query->comment, std::make_shared(comment_value)); diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp index 165bad950f5..9f56b5f7676 100644 --- a/src/Databases/DatabaseOnDisk.cpp +++ b/src/Databases/DatabaseOnDisk.cpp @@ -77,6 +77,10 @@ std::pair createTableFromAST( /// - the code is simpler, since the query is already brought to a suitable form. if (!ast_create_query.columns_list || !ast_create_query.columns_list->columns) { + if (!ast_create_query.storage || !ast_create_query.storage->engine) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid storage definition in metadata file: " + "it's a bug or result of manual intervention in metadata files"); + if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(ast_create_query.storage->engine->name)) throw Exception("Missing definition of columns.", ErrorCodes::EMPTY_LIST_OF_COLUMNS_PASSED); /// Leave columns empty. @@ -316,7 +320,7 @@ void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_na } catch (...) { - LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__)); + LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(__PRETTY_FUNCTION__))); attachTable(local_context, table_name, table, table_data_path_relative); if (renamed) fs::rename(table_metadata_path_drop, table_metadata_path); diff --git a/src/Databases/DatabaseOnDisk.h b/src/Databases/DatabaseOnDisk.h index 9bc8fa3bcef..64122ae66e5 100644 --- a/src/Databases/DatabaseOnDisk.h +++ b/src/Databases/DatabaseOnDisk.h @@ -69,7 +69,7 @@ public: static ASTPtr parseQueryFromMetadata(Poco::Logger * log, ContextPtr context, const String & metadata_file_path, bool throw_on_error = true, bool remove_empty = false); /// will throw when the table we want to attach already exists (in active / detached / detached permanently form) - void checkMetadataFilenameAvailability(const String & to_table_name) const; + void checkMetadataFilenameAvailability(const String & to_table_name) const override; void checkMetadataFilenameAvailabilityUnlocked(const String & to_table_name, std::unique_lock &) const; void modifySettingsMetadata(const SettingsChanges & settings_changes, ContextPtr query_context); diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index 84d2edd1bb1..d9d9f5b45f6 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -142,7 +142,7 @@ ClusterPtr DatabaseReplicated::getClusterImpl() const "It's possible if the first replica is not fully created yet " "or if the last replica was just dropped or due to logical error", database_name); Int32 cversion = stat.cversion; - std::sort(hosts.begin(), hosts.end()); + ::sort(hosts.begin(), hosts.end()); std::vector futures; futures.reserve(hosts.size()); diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h index b1aa4eb1aae..9ad33bd228f 100644 --- a/src/Databases/IDatabase.h +++ b/src/Databases/IDatabase.h @@ -158,9 +158,14 @@ public: virtual void startupTables(ThreadPool & /*thread_pool*/, bool /*force_restore*/, bool /*force_attach*/) {} - /// Check the existence of the table. + /// Check the existence of the table in memory (attached). virtual bool isTableExist(const String & name, ContextPtr context) const = 0; + /// Check the existence of the table in any state (in active / detached / detached permanently state). + /// Throws exception when table exists. + virtual void checkMetadataFilenameAvailability(const String & /*table_name*/) const {} + + /// Get the table for work. Return nullptr if there is no table. virtual StoragePtr tryGetTable(const String & name, ContextPtr context) const = 0; diff --git a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp index 9dbe611537b..8033d65c549 100644 --- a/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp +++ b/src/Databases/MySQL/MaterializedMySQLSyncThread.cpp @@ -316,7 +316,7 @@ getTableOutput(const String & database_name, const String & table_name, ContextM return std::move(res.pipeline); } -static inline String reWriteMysqlQueryColumn(mysqlxx::Pool::Entry & connection, const String & database_name, const String & table_name, const Settings & global_settings) +static inline String rewriteMysqlQueryColumn(mysqlxx::Pool::Entry & connection, const String & database_name, const String & table_name, const Settings & global_settings) { Block tables_columns_sample_block { @@ -376,7 +376,7 @@ static inline void dumpDataForTables( auto pipeline = getTableOutput(database_name, table_name, query_context); StreamSettings mysql_input_stream_settings(context->getSettingsRef()); - String mysql_select_all_query = "SELECT " + reWriteMysqlQueryColumn(connection, mysql_database_name, table_name, context->getSettings()) + " FROM " + String mysql_select_all_query = "SELECT " + rewriteMysqlQueryColumn(connection, mysql_database_name, table_name, context->getSettingsRef()) + " FROM " + backQuoteIfNeed(mysql_database_name) + "." + backQuoteIfNeed(table_name); LOG_INFO(&Poco::Logger::get("MaterializedMySQLSyncThread(" + database_name + ")"), "mysql_select_all_query is {}", mysql_select_all_query); auto input = std::make_unique(connection, mysql_select_all_query, pipeline.getHeader(), mysql_input_stream_settings); diff --git a/src/Databases/SQLite/DatabaseSQLite.cpp b/src/Databases/SQLite/DatabaseSQLite.cpp index f4dab8a91a8..fb742587285 100644 --- a/src/Databases/SQLite/DatabaseSQLite.cpp +++ b/src/Databases/SQLite/DatabaseSQLite.cpp @@ -94,7 +94,7 @@ bool DatabaseSQLite::checkSQLiteTable(const String & table_name) const if (!sqlite_db) sqlite_db = openSQLiteDB(database_path, getContext(), /* throw_on_error */true); - const String query = fmt::format("SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}';", table_name); + const String query = fmt::format("SELECT name FROM sqlite_master WHERE type='table' AND name='{}';", table_name); auto callback_get_data = [](void * res, int, char **, char **) -> int { diff --git a/src/Databases/SQLite/SQLiteUtils.cpp b/src/Databases/SQLite/SQLiteUtils.cpp index 954576d9c05..5b38caeabee 100644 --- a/src/Databases/SQLite/SQLiteUtils.cpp +++ b/src/Databases/SQLite/SQLiteUtils.cpp @@ -20,7 +20,7 @@ void processSQLiteError(const String & message, bool throw_on_error) if (throw_on_error) throw Exception(ErrorCodes::PATH_ACCESS_DENIED, message); else - LOG_ERROR(&Poco::Logger::get("SQLiteEngine"), message); + LOG_ERROR(&Poco::Logger::get("SQLiteEngine"), fmt::runtime(message)); } diff --git a/src/Dictionaries/CacheDictionaryStorage.h b/src/Dictionaries/CacheDictionaryStorage.h index d6d04075a3d..566515c7cc8 100644 --- a/src/Dictionaries/CacheDictionaryStorage.h +++ b/src/Dictionaries/CacheDictionaryStorage.h @@ -8,10 +8,10 @@ #include #include #include +#include #include #include #include -#include namespace DB diff --git a/src/Dictionaries/CassandraHelpers.cpp b/src/Dictionaries/CassandraHelpers.cpp index a33ab288a34..235e29b5bd8 100644 --- a/src/Dictionaries/CassandraHelpers.cpp +++ b/src/Dictionaries/CassandraHelpers.cpp @@ -58,15 +58,15 @@ void cassandraLogCallback(const CassLogMessage * message, void * data) { Poco::Logger * logger = static_cast(data); if (message->severity == CASS_LOG_CRITICAL || message->severity == CASS_LOG_ERROR) - LOG_ERROR(logger, message->message); + LOG_ERROR(logger, fmt::runtime(message->message)); else if (message->severity == CASS_LOG_WARN) - LOG_WARNING(logger, message->message); + LOG_WARNING(logger, fmt::runtime(message->message)); else if (message->severity == CASS_LOG_INFO) - LOG_INFO(logger, message->message); + LOG_INFO(logger, fmt::runtime(message->message)); else if (message->severity == CASS_LOG_DEBUG) - LOG_DEBUG(logger, message->message); + LOG_DEBUG(logger, fmt::runtime(message->message)); else if (message->severity == CASS_LOG_TRACE) - LOG_TRACE(logger, message->message); + LOG_TRACE(logger, fmt::runtime(message->message)); } } diff --git a/src/Dictionaries/DictionaryHelpers.h b/src/Dictionaries/DictionaryHelpers.h index 5c2b6b27afd..f2d7febfa8e 100644 --- a/src/Dictionaries/DictionaryHelpers.h +++ b/src/Dictionaries/DictionaryHelpers.h @@ -623,17 +623,6 @@ void mergeBlockWithPipe( } } -template -static StringRef copyStringInArena(Arena & arena, StringRef value) -{ - size_t key_size = value.size; - char * place_for_key = arena.alloc(key_size); - memcpy(reinterpret_cast(place_for_key), reinterpret_cast(value.data), key_size); - StringRef result{place_for_key, key_size}; - - return result; -} - /** * Returns ColumnVector data as PaddedPodArray. diff --git a/src/Dictionaries/ExecutablePoolDictionarySource.cpp b/src/Dictionaries/ExecutablePoolDictionarySource.cpp index 48ddeed7fa6..62598c966e5 100644 --- a/src/Dictionaries/ExecutablePoolDictionarySource.cpp +++ b/src/Dictionaries/ExecutablePoolDictionarySource.cpp @@ -197,7 +197,7 @@ void registerDictionarySourceExecutablePool(DictionarySourceFactory & factory) size_t max_command_execution_time = config.getUInt64(settings_config_prefix + ".max_command_execution_time", 10); - size_t max_execution_time_seconds = static_cast(context->getSettings().max_execution_time.totalSeconds()); + size_t max_execution_time_seconds = static_cast(context->getSettingsRef().max_execution_time.totalSeconds()); if (max_execution_time_seconds != 0 && max_command_execution_time > max_execution_time_seconds) max_command_execution_time = max_execution_time_seconds; diff --git a/src/Dictionaries/FlatDictionary.cpp b/src/Dictionaries/FlatDictionary.cpp index 40cc735557c..0c82da7b73b 100644 --- a/src/Dictionaries/FlatDictionary.cpp +++ b/src/Dictionaries/FlatDictionary.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -13,7 +14,7 @@ #include #include -#include +#include #include #include diff --git a/src/Dictionaries/HashedArrayDictionary.cpp b/src/Dictionaries/HashedArrayDictionary.cpp index e35340c7618..ea041c63d73 100644 --- a/src/Dictionaries/HashedArrayDictionary.cpp +++ b/src/Dictionaries/HashedArrayDictionary.cpp @@ -1,5 +1,6 @@ #include "HashedArrayDictionary.h" +#include #include #include #include diff --git a/src/Dictionaries/HashedDictionary.cpp b/src/Dictionaries/HashedDictionary.cpp index c83735a6330..b70f018df6b 100644 --- a/src/Dictionaries/HashedDictionary.cpp +++ b/src/Dictionaries/HashedDictionary.cpp @@ -1,5 +1,6 @@ #include "HashedDictionary.h" +#include #include #include #include diff --git a/src/Dictionaries/IPAddressDictionary.cpp b/src/Dictionaries/IPAddressDictionary.cpp index 9945ee1d4b3..929b04d14fa 100644 --- a/src/Dictionaries/IPAddressDictionary.cpp +++ b/src/Dictionaries/IPAddressDictionary.cpp @@ -13,10 +13,12 @@ #include #include #include +#include #include #include #include + namespace DB { namespace ErrorCodes @@ -145,7 +147,7 @@ static void validateKeyTypes(const DataTypes & key_types) template size_t sortAndUnique(std::vector & vec, Comp comp) { - std::sort(vec.begin(), vec.end(), + ::sort(vec.begin(), vec.end(), [&](const auto & a, const auto & b) { return comp(a, b) < 0; }); auto new_end = std::unique(vec.begin(), vec.end(), diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index a291fcea47f..29d70f3a7c4 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -193,7 +193,7 @@ Pipe MySQLDictionarySource::loadAll() auto connection = pool->get(); last_modification = getLastModification(connection, false); - LOG_TRACE(log, load_all_query); + LOG_TRACE(log, fmt::runtime(load_all_query)); return loadFromQuery(load_all_query); } @@ -203,7 +203,7 @@ Pipe MySQLDictionarySource::loadUpdatedAll() last_modification = getLastModification(connection, false); std::string load_update_query = getUpdateFieldAndDate(); - LOG_TRACE(log, load_update_query); + LOG_TRACE(log, fmt::runtime(load_update_query)); return loadFromQuery(load_update_query); } @@ -289,7 +289,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry & { auto query = connection->query("SHOW TABLE STATUS LIKE " + quoteForLike(configuration.table)); - LOG_TRACE(log, query.str()); + LOG_TRACE(log, fmt::runtime(query.str())); auto result = query.use(); diff --git a/src/Dictionaries/PolygonDictionary.cpp b/src/Dictionaries/PolygonDictionary.cpp index 8aa9527e467..deec1e6a588 100644 --- a/src/Dictionaries/PolygonDictionary.cpp +++ b/src/Dictionaries/PolygonDictionary.cpp @@ -3,6 +3,8 @@ #include #include +#include + #include #include #include @@ -250,7 +252,7 @@ void IPolygonDictionary::loadData() polygon_ids.emplace_back(polygon, i); } - std::sort(polygon_ids.begin(), polygon_ids.end(), [& areas](const auto & lhs, const auto & rhs) + ::sort(polygon_ids.begin(), polygon_ids.end(), [& areas](const auto & lhs, const auto & rhs) { return areas[lhs.second] < areas[rhs.second]; }); diff --git a/src/Dictionaries/PolygonDictionaryUtils.cpp b/src/Dictionaries/PolygonDictionaryUtils.cpp index 15267481c0b..804920f85e6 100644 --- a/src/Dictionaries/PolygonDictionaryUtils.cpp +++ b/src/Dictionaries/PolygonDictionaryUtils.cpp @@ -3,11 +3,13 @@ #include #include +#include #include #include #include + namespace DB { @@ -87,7 +89,7 @@ std::vector SlabsPolygonIndex::uniqueX(const std::vector & polyg } /** Making all_x sorted and distinct */ - std::sort(all_x.begin(), all_x.end()); + ::sort(all_x.begin(), all_x.end()); all_x.erase(std::unique(all_x.begin(), all_x.end()), all_x.end()); return all_x; @@ -104,7 +106,7 @@ void SlabsPolygonIndex::indexBuild(const std::vector & polygons) } /** Sorting edges of (left_point, right_point, polygon_id) in that order */ - std::sort(all_edges.begin(), all_edges.end(), Edge::compareByLeftPoint); + ::sort(all_edges.begin(), all_edges.end(), Edge::compareByLeftPoint); for (size_t i = 0; i != all_edges.size(); ++i) all_edges[i].edge_id = i; @@ -298,7 +300,7 @@ bool SlabsPolygonIndex::find(const Point & point, size_t & id) const } while (pos != 0); /** Sort all ids and find smallest with odd occurrences */ - std::sort(intersections.begin(), intersections.end()); + ::sort(intersections.begin(), intersections.end()); for (size_t i = 0; i < intersections.size(); i += 2) { if (i + 1 == intersections.size() || intersections[i] != intersections[i + 1]) diff --git a/src/Dictionaries/PostgreSQLDictionarySource.cpp b/src/Dictionaries/PostgreSQLDictionarySource.cpp index 9af3ea06838..6fdf486fdbf 100644 --- a/src/Dictionaries/PostgreSQLDictionarySource.cpp +++ b/src/Dictionaries/PostgreSQLDictionarySource.cpp @@ -80,7 +80,7 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource(const PostgreSQLDictionar Pipe PostgreSQLDictionarySource::loadAll() { - LOG_TRACE(log, load_all_query); + LOG_TRACE(log, fmt::runtime(load_all_query)); return loadBase(load_all_query); } @@ -88,7 +88,7 @@ Pipe PostgreSQLDictionarySource::loadAll() Pipe PostgreSQLDictionarySource::loadUpdatedAll() { auto load_update_query = getUpdateFieldAndDate(); - LOG_TRACE(log, load_update_query); + LOG_TRACE(log, fmt::runtime(load_update_query)); return loadBase(load_update_query); } diff --git a/src/Dictionaries/RangeHashedDictionary.cpp b/src/Dictionaries/RangeHashedDictionary.cpp index 14c8fc7c749..5330bc684c3 100644 --- a/src/Dictionaries/RangeHashedDictionary.cpp +++ b/src/Dictionaries/RangeHashedDictionary.cpp @@ -1,5 +1,7 @@ #include +#include + #include #include #include diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index 7c7dc838436..adbe4084d81 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -12,9 +12,11 @@ #include #include +#include #include #include #include +#include #include #include #include @@ -24,6 +26,7 @@ #include #include + namespace CurrentMetrics { extern const Metric Write; @@ -1092,7 +1095,7 @@ private: } /// Sort blocks by offset before start async io requests - std::sort(blocks_to_request.begin(), blocks_to_request.end()); + ::sort(blocks_to_request.begin(), blocks_to_request.end()); file_buffer.fetchBlocks(configuration.read_buffer_blocks_size, blocks_to_request, [&](size_t block_index, char * block_data) { diff --git a/src/Dictionaries/XDBCDictionarySource.cpp b/src/Dictionaries/XDBCDictionarySource.cpp index e95094cac47..f08abcdc516 100644 --- a/src/Dictionaries/XDBCDictionarySource.cpp +++ b/src/Dictionaries/XDBCDictionarySource.cpp @@ -121,7 +121,7 @@ std::string XDBCDictionarySource::getUpdateFieldAndDate() Pipe XDBCDictionarySource::loadAll() { - LOG_TRACE(log, load_all_query); + LOG_TRACE(log, fmt::runtime(load_all_query)); return loadFromQuery(bridge_url, sample_block, load_all_query); } @@ -130,7 +130,7 @@ Pipe XDBCDictionarySource::loadUpdatedAll() { std::string load_query_update = getUpdateFieldAndDate(); - LOG_TRACE(log, load_query_update); + LOG_TRACE(log, fmt::runtime(load_query_update)); return loadFromQuery(bridge_url, sample_block, load_query_update); } diff --git a/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp b/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp index 31e85442c6a..a1743720f47 100644 --- a/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp +++ b/src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp @@ -66,7 +66,7 @@ std::unique_ptr DiskAzureBlobStorage::readFile( std::optional) const { auto settings = current_settings.get(); - auto metadata = readMeta(path); + auto metadata = readMetadata(path); LOG_TEST(log, "Read from file by path: {}", backQuote(metadata_disk->getPath() + path)); @@ -94,7 +94,6 @@ std::unique_ptr DiskAzureBlobStorage::writeFile( size_t buf_size, WriteMode mode) { - auto metadata = readOrCreateMetaForWriting(path, mode); auto blob_path = path + "_" + getRandomASCIIString(8); /// NOTE: path contains the tmp_* prefix in the blob name LOG_TRACE(log, "{} to file by path: {}. AzureBlob Storage path: {}", @@ -106,7 +105,12 @@ std::unique_ptr DiskAzureBlobStorage::writeFile( current_settings.get()->max_single_part_upload_size, buf_size); - return std::make_unique>(std::move(buffer), std::move(metadata), blob_path); + auto create_metadata_callback = [this, path, mode, blob_path] (size_t count) + { + readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_path, count] (Metadata & metadata) { metadata.addObject(blob_path, count); return true; }); + }; + + return std::make_unique>(std::move(buffer), std::move(create_metadata_callback), path); } diff --git a/src/Disks/DiskCacheWrapper.cpp b/src/Disks/DiskCacheWrapper.cpp index 46ea46f85ef..5d5eb89691e 100644 --- a/src/Disks/DiskCacheWrapper.cpp +++ b/src/Disks/DiskCacheWrapper.cpp @@ -8,15 +8,22 @@ namespace DB { /** - * Write buffer with possibility to set and invoke callback after 'finalize' call. + * This buffer writes to cache, but after finalize() copy written file from cache to disk. */ -class CompletionAwareWriteBuffer : public WriteBufferFromFileDecorator +class WritingToCacheWriteBuffer final : public WriteBufferFromFileDecorator { public: - CompletionAwareWriteBuffer(std::unique_ptr impl_, std::function completion_callback_) - : WriteBufferFromFileDecorator(std::move(impl_)), completion_callback(completion_callback_) { } + WritingToCacheWriteBuffer( + std::unique_ptr impl_, + std::function()> create_read_buffer_, + std::function()> create_write_buffer_) + : WriteBufferFromFileDecorator(std::move(impl_)) + , create_read_buffer(std::move(create_read_buffer_)) + , create_write_buffer(std::move(create_write_buffer_)) + { + } - virtual ~CompletionAwareWriteBuffer() override + ~WritingToCacheWriteBuffer() override { try { @@ -28,15 +35,36 @@ public: } } + void preFinalize() override + { + impl->next(); + impl->preFinalize(); + impl->finalize(); + + read_buffer = create_read_buffer(); + write_buffer = create_write_buffer(); + copyData(*read_buffer, *write_buffer); + write_buffer->next(); + write_buffer->preFinalize(); + + is_prefinalized = true; + } + void finalizeImpl() override { - WriteBufferFromFileDecorator::finalizeImpl(); + if (!is_prefinalized) + preFinalize(); - completion_callback(); + write_buffer->finalize(); } private: - const std::function completion_callback; + std::function()> create_read_buffer; + std::function()> create_write_buffer; + std::unique_ptr read_buffer; + std::unique_ptr write_buffer; + + bool is_prefinalized = false; }; enum FileDownloadStatus @@ -165,21 +193,22 @@ DiskCacheWrapper::writeFile(const String & path, size_t buf_size, WriteMode mode if (!cache_file_predicate(path)) return DiskDecorator::writeFile(path, buf_size, mode); - LOG_TRACE(log, "Write file {} to cache", backQuote(path)); + LOG_TEST(log, "Write file {} to cache", backQuote(path)); auto dir_path = directoryPath(path); if (!cache_disk->exists(dir_path)) cache_disk->createDirectories(dir_path); - return std::make_unique( + return std::make_unique( cache_disk->writeFile(path, buf_size, mode), - [this, path, buf_size, mode]() + [this, path]() { /// Copy file from cache to actual disk when cached buffer is finalized. - auto src_buffer = cache_disk->readFile(path, ReadSettings(), /* read_hint= */ {}, /* file_size= */ {}); - auto dst_buffer = DiskDecorator::writeFile(path, buf_size, mode); - copyData(*src_buffer, *dst_buffer); - dst_buffer->finalize(); + return cache_disk->readFile(path, ReadSettings(), /* read_hint= */ {}, /* file_size= */ {}); + }, + [this, path, buf_size, mode]() + { + return DiskDecorator::writeFile(path, buf_size, mode); }); } @@ -245,6 +274,7 @@ void DiskCacheWrapper::removeDirectory(const String & path) { if (cache_disk->exists(path)) cache_disk->removeDirectory(path); + DiskDecorator::removeDirectory(path); } @@ -269,6 +299,18 @@ void DiskCacheWrapper::removeSharedRecursive(const String & path, bool keep_s3) DiskDecorator::removeSharedRecursive(path, keep_s3); } + +void DiskCacheWrapper::removeSharedFiles(const RemoveBatchRequest & files, bool keep_s3) +{ + for (const auto & file : files) + { + if (cache_disk->exists(file.path)) + cache_disk->removeSharedFile(file.path, keep_s3); + } + + DiskDecorator::removeSharedFiles(files, keep_s3); +} + void DiskCacheWrapper::createHardLink(const String & src_path, const String & dst_path) { /// Don't create hardlinks for cache files to shadow directory as it just waste cache disk space. diff --git a/src/Disks/DiskCacheWrapper.h b/src/Disks/DiskCacheWrapper.h index 6eb79114a54..dc66333758f 100644 --- a/src/Disks/DiskCacheWrapper.h +++ b/src/Disks/DiskCacheWrapper.h @@ -48,6 +48,7 @@ public: void removeRecursive(const String & path) override; void removeSharedFile(const String & path, bool keep_s3) override; void removeSharedRecursive(const String & path, bool keep_s3) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool keep_s3) override; void createHardLink(const String & src_path, const String & dst_path) override; ReservationPtr reserve(UInt64 bytes) override; diff --git a/src/Disks/DiskDecorator.cpp b/src/Disks/DiskDecorator.cpp index d4acb6fab0d..37911f16913 100644 --- a/src/Disks/DiskDecorator.cpp +++ b/src/Disks/DiskDecorator.cpp @@ -151,6 +151,11 @@ void DiskDecorator::removeSharedFile(const String & path, bool keep_s3) delegate->removeSharedFile(path, keep_s3); } +void DiskDecorator::removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) +{ + delegate->removeSharedFiles(files, keep_in_remote_fs); +} + void DiskDecorator::removeSharedRecursive(const String & path, bool keep_s3) { delegate->removeSharedRecursive(path, keep_s3); diff --git a/src/Disks/DiskDecorator.h b/src/Disks/DiskDecorator.h index ff4f16fdf3d..bace54ff22a 100644 --- a/src/Disks/DiskDecorator.h +++ b/src/Disks/DiskDecorator.h @@ -52,6 +52,7 @@ public: void removeRecursive(const String & path) override; void removeSharedFile(const String & path, bool keep_s3) override; void removeSharedRecursive(const String & path, bool keep_s3) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) override; void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; Poco::Timestamp getLastModified(const String & path) override; void setReadOnly(const String & path) override; @@ -71,17 +72,9 @@ public: void startup() override; void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap & map) override; - std::unique_ptr readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const override { return delegate->readMetaFile(path, settings, size); } + DiskPtr getMetadataDiskIfExistsOrSelf() override { return delegate->getMetadataDiskIfExistsOrSelf(); } - std::unique_ptr writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode) override { return delegate->writeMetaFile(path, buf_size, mode); } - - void removeMetaFileIfExists(const String & path) override { delegate->removeMetaFileIfExists(path); } + std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override { return delegate->getSerializedMetadata(file_paths); } UInt32 getRefCount(const String & path) const override { return delegate->getRefCount(path); } diff --git a/src/Disks/DiskLocal.cpp b/src/Disks/DiskLocal.cpp index caa8d44025d..57bfaf405e0 100644 --- a/src/Disks/DiskLocal.cpp +++ b/src/Disks/DiskLocal.cpp @@ -11,6 +11,14 @@ #include #include +#include +#include +#include +#include +#include +#include +#include +#include namespace CurrentMetrics { @@ -25,7 +33,7 @@ namespace ErrorCodes extern const int UNKNOWN_ELEMENT_IN_CONFIG; extern const int EXCESSIVE_ELEMENT_IN_CONFIG; extern const int PATH_ACCESS_DENIED; - extern const int INCORRECT_DISK_INDEX; + extern const int LOGICAL_ERROR; extern const int CANNOT_TRUNCATE_FILE; extern const int CANNOT_UNLINK; extern const int CANNOT_RMDIR; @@ -61,9 +69,6 @@ static void loadDiskLocalConfig(const String & name, throw Exception("Disk path must end with /. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); } - if (!FS::canRead(path) || !FS::canWrite(path)) - throw Exception("There is no RW access to the disk " + name + " (" + path + ")", ErrorCodes::PATH_ACCESS_DENIED); - bool has_space_ratio = config.has(config_prefix + ".keep_free_space_ratio"); if (config.has(config_prefix + ".keep_free_space_bytes") && has_space_ratio) @@ -113,13 +118,48 @@ public: UInt64 getSize() const override { return size; } - DiskPtr getDisk(size_t i) const override; + DiskPtr getDisk(size_t i) const override + { + if (i != 0) + throw Exception("Can't use i != 0 with single disk reservation. It's a bug", ErrorCodes::LOGICAL_ERROR); + return disk; + } Disks getDisks() const override { return {disk}; } - void update(UInt64 new_size) override; + void update(UInt64 new_size) override + { + std::lock_guard lock(DiskLocal::reservation_mutex); + disk->reserved_bytes -= size; + size = new_size; + disk->reserved_bytes += size; + } - ~DiskLocalReservation() override; + ~DiskLocalReservation() override + { + try + { + std::lock_guard lock(DiskLocal::reservation_mutex); + if (disk->reserved_bytes < size) + { + disk->reserved_bytes = 0; + LOG_ERROR(&Poco::Logger::get("DiskLocal"), "Unbalanced reservations size for disk '{}'.", disk->getName()); + } + else + { + disk->reserved_bytes -= size; + } + + if (disk->reservation_count == 0) + LOG_ERROR(&Poco::Logger::get("DiskLocal"), "Unbalanced reservation count for disk '{}'.", disk->getName()); + else + --disk->reservation_count; + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } private: DiskLocalPtr disk; @@ -188,7 +228,7 @@ bool DiskLocal::tryReserve(UInt64 bytes) return false; } -UInt64 DiskLocal::getTotalSpace() const +static UInt64 getTotalSpaceByName(const String & name, const String & disk_path, UInt64 keep_free_space_bytes) { struct statvfs fs; if (name == "default") /// for default disk we get space from path/data/ @@ -201,8 +241,17 @@ UInt64 DiskLocal::getTotalSpace() const return total_size - keep_free_space_bytes; } +UInt64 DiskLocal::getTotalSpace() const +{ + if (broken || readonly) + return 0; + return getTotalSpaceByName(name, disk_path, keep_free_space_bytes); +} + UInt64 DiskLocal::getAvailableSpace() const { + if (broken || readonly) + return 0; /// we use f_bavail, because part of b_free space is /// available for superuser only and for system purposes struct statvfs fs; @@ -268,7 +317,7 @@ void DiskLocal::moveDirectory(const String & from_path, const String & to_path) DiskDirectoryIteratorPtr DiskLocal::iterateDirectory(const String & path) { fs::path meta_path = fs::path(disk_path) / path; - if (fs::exists(meta_path) && fs::is_directory(meta_path)) + if (!broken && fs::exists(meta_path) && fs::is_directory(meta_path)) return std::make_unique(disk_path, path); else return std::make_unique(); @@ -409,49 +458,191 @@ void DiskLocal::applyNewSettings(const Poco::Util::AbstractConfiguration & confi keep_free_space_bytes = new_keep_free_space_bytes; } -DiskPtr DiskLocalReservation::getDisk(size_t i) const +DiskLocal::DiskLocal(const String & name_, const String & path_, UInt64 keep_free_space_bytes_) + : name(name_) + , disk_path(path_) + , keep_free_space_bytes(keep_free_space_bytes_) + , logger(&Poco::Logger::get("DiskLocal")) { - if (i != 0) - { - throw Exception("Can't use i != 0 with single disk reservation", ErrorCodes::INCORRECT_DISK_INDEX); - } - return disk; } -void DiskLocalReservation::update(UInt64 new_size) +DiskLocal::DiskLocal( + const String & name_, const String & path_, UInt64 keep_free_space_bytes_, ContextPtr context, UInt64 local_disk_check_period_ms) + : DiskLocal(name_, path_, keep_free_space_bytes_) { - std::lock_guard lock(DiskLocal::reservation_mutex); - disk->reserved_bytes -= size; - size = new_size; - disk->reserved_bytes += size; + if (local_disk_check_period_ms > 0) + disk_checker = std::make_unique(this, context, local_disk_check_period_ms); } -DiskLocalReservation::~DiskLocalReservation() +void DiskLocal::startup() { try { - std::lock_guard lock(DiskLocal::reservation_mutex); - if (disk->reserved_bytes < size) - { - disk->reserved_bytes = 0; - LOG_ERROR(disk->log, "Unbalanced reservations size for disk '{}'.", disk->getName()); - } - else - { - disk->reserved_bytes -= size; - } - - if (disk->reservation_count == 0) - LOG_ERROR(disk->log, "Unbalanced reservation count for disk '{}'.", disk->getName()); - else - --disk->reservation_count; + broken = false; + disk_checker_magic_number = -1; + disk_checker_can_check_read = true; + readonly = !setup(); } catch (...) { - tryLogCurrentException(__PRETTY_FUNCTION__); + tryLogCurrentException(logger, fmt::format("Disk {} is marked as broken during startup", name)); + broken = true; + /// Disk checker is disabled when failing to start up. + disk_checker_can_check_read = false; } + if (disk_checker && disk_checker_can_check_read) + disk_checker->startup(); } +void DiskLocal::shutdown() +{ + if (disk_checker) + disk_checker->shutdown(); +} + +std::optional DiskLocal::readDiskCheckerMagicNumber() const noexcept +try +{ + ReadSettings read_settings; + /// Proper disk read checking requires direct io + read_settings.direct_io_threshold = 1; + auto buf = readFile(disk_checker_path, read_settings, {}, {}); + UInt32 magic_number; + readIntBinary(magic_number, *buf); + if (buf->eof()) + return magic_number; + LOG_WARNING(logger, "The size of disk check magic number is more than 4 bytes. Mark it as read failure"); + return {}; +} +catch (...) +{ + tryLogCurrentException(logger, fmt::format("Cannot read correct disk check magic number from from {}{}", disk_path, disk_checker_path)); + return {}; +} + +bool DiskLocal::canRead() const noexcept +try +{ + if (FS::canRead(fs::path(disk_path) / disk_checker_path)) + { + auto magic_number = readDiskCheckerMagicNumber(); + if (magic_number && *magic_number == disk_checker_magic_number) + return true; + } + return false; +} +catch (...) +{ + LOG_WARNING(logger, "Cannot achieve read over the disk directory: {}", disk_path); + return false; +} + +struct DiskWriteCheckData +{ + constexpr static size_t PAGE_SIZE_IN_BYTES = 4096; + char data[PAGE_SIZE_IN_BYTES]{}; + DiskWriteCheckData() + { + static const char * magic_string = "ClickHouse disk local write check"; + static size_t magic_string_len = strlen(magic_string); + memcpy(data, magic_string, magic_string_len); + memcpy(data + PAGE_SIZE_IN_BYTES - magic_string_len, magic_string, magic_string_len); + } +}; + +bool DiskLocal::canWrite() const noexcept +try +{ + static DiskWriteCheckData data; + String tmp_template = fs::path(disk_path) / ""; + { + auto buf = WriteBufferFromTemporaryFile::create(tmp_template); + buf->write(data.data, data.PAGE_SIZE_IN_BYTES); + buf->sync(); + } + return true; +} +catch (...) +{ + LOG_WARNING(logger, "Cannot achieve write over the disk directory: {}", disk_path); + return false; +} + +bool DiskLocal::setup() +{ + try + { + fs::create_directories(disk_path); + } + catch (...) + { + LOG_ERROR(logger, "Cannot create the directory of disk {} ({}).", name, disk_path); + throw; + } + + try + { + if (!FS::canRead(disk_path)) + throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "There is no read access to disk {} ({}).", name, disk_path); + } + catch (...) + { + LOG_ERROR(logger, "Cannot gain read access of the disk directory: {}", disk_path); + throw; + } + + /// If disk checker is disabled, just assume RW by default. + if (!disk_checker) + return true; + + try + { + if (exists(disk_checker_path)) + { + auto magic_number = readDiskCheckerMagicNumber(); + if (magic_number) + disk_checker_magic_number = *magic_number; + else + { + /// The checker file is incorrect. Mark the magic number to uninitialized and try to generate a new checker file. + disk_checker_magic_number = -1; + } + } + } + catch (...) + { + LOG_ERROR(logger, "We cannot tell if {} exists anymore, or read from it. Most likely disk {} is broken", disk_checker_path, name); + throw; + } + + /// Try to create a new checker file. The disk status can be either broken or readonly. + if (disk_checker_magic_number == -1) + try + { + pcg32_fast rng(randomSeed()); + UInt32 magic_number = rng(); + { + auto buf = writeFile(disk_checker_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); + writeIntBinary(magic_number, *buf); + } + disk_checker_magic_number = magic_number; + } + catch (...) + { + LOG_WARNING( + logger, + "Cannot create/write to {0}. Disk {1} is either readonly or broken. Without setting up disk checker file, DiskLocalCheckThread " + "will not be started. Disk is assumed to be RW. Try manually fix the disk and do `SYSTEM RESTART DISK {1}`", + disk_checker_path, + name); + disk_checker_can_check_read = false; + return true; + } + + if (disk_checker_magic_number == -1) + throw Exception("disk_checker_magic_number is not initialized. It's a bug", ErrorCodes::LOGICAL_ERROR); + return true; +} void registerDiskLocal(DiskFactory & factory) { @@ -459,17 +650,20 @@ void registerDiskLocal(DiskFactory & factory) const Poco::Util::AbstractConfiguration & config, const String & config_prefix, ContextPtr context, - const DisksMap & map) -> DiskPtr { + const DisksMap & map) -> DiskPtr + { String path; UInt64 keep_free_space_bytes; loadDiskLocalConfig(name, config, config_prefix, context, path, keep_free_space_bytes); for (const auto & [disk_name, disk_ptr] : map) - { if (path == disk_ptr->getPath()) - throw Exception("Disk " + name + " and Disk " + disk_name + " cannot have the same path" + " (" + path + ")", ErrorCodes::BAD_ARGUMENTS); - } - return std::make_shared(name, path, keep_free_space_bytes); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk {} and disk {} cannot have the same path ({})", name, disk_name, path); + + std::shared_ptr disk + = std::make_shared(name, path, keep_free_space_bytes, context, config.getUInt("local_disk_check_period_ms", 0)); + disk->startup(); + return std::make_shared(disk); }; factory.registerDiskType("local", creator); } diff --git a/src/Disks/DiskLocal.h b/src/Disks/DiskLocal.h index f16497ae432..76d5a88a626 100644 --- a/src/Disks/DiskLocal.h +++ b/src/Disks/DiskLocal.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -10,24 +11,22 @@ namespace DB { -namespace ErrorCodes -{ - extern const int LOGICAL_ERROR; -} class DiskLocalReservation; class DiskLocal : public IDisk { public: + friend class DiskLocalCheckThread; friend class DiskLocalReservation; - DiskLocal(const String & name_, const String & path_, UInt64 keep_free_space_bytes_) - : name(name_), disk_path(path_), keep_free_space_bytes(keep_free_space_bytes_) - { - if (disk_path.back() != '/') - throw Exception("Disk path must end with '/', but '" + disk_path + "' doesn't.", ErrorCodes::LOGICAL_ERROR); - } + DiskLocal(const String & name_, const String & path_, UInt64 keep_free_space_bytes_); + DiskLocal( + const String & name_, + const String & path_, + UInt64 keep_free_space_bytes_, + ContextPtr context, + UInt64 local_disk_check_period_ms); const String & getName() const override { return name; } @@ -106,13 +105,33 @@ public: void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String & config_prefix, const DisksMap &) override; + bool isBroken() const override { return broken; } + + void startup() override; + + void shutdown() override; + + /// Check if the disk is OK to proceed read/write operations. Currently the check is + /// rudimentary. The more advanced choice would be using + /// https://github.com/smartmontools/smartmontools. However, it's good enough for now. + bool canRead() const noexcept; + bool canWrite() const noexcept; + private: bool tryReserve(UInt64 bytes); -private: + /// Setup disk for healthy check. Returns true if it's read-write, false if read-only. + /// Throw exception if it's not possible to setup necessary files and directories. + bool setup(); + + /// Read magic number from disk checker file. Return std::nullopt if exception happens. + std::optional readDiskCheckerMagicNumber() const noexcept; + const String name; const String disk_path; + const String disk_checker_path = ".disk_checker_file"; std::atomic keep_free_space_bytes; + Poco::Logger * logger; UInt64 reserved_bytes = 0; UInt64 reservation_count = 0; @@ -120,6 +139,14 @@ private: static std::mutex reservation_mutex; Poco::Logger * log = &Poco::Logger::get("DiskLocal"); + + std::atomic broken{false}; + std::atomic readonly{false}; + std::unique_ptr disk_checker; + /// A magic number to vaguely check if reading operation generates correct result. + /// -1 means there is no available disk_checker_file yet. + Int64 disk_checker_magic_number = -1; + bool disk_checker_can_check_read = true; }; diff --git a/src/Disks/DiskLocalCheckThread.cpp b/src/Disks/DiskLocalCheckThread.cpp new file mode 100644 index 00000000000..9ebedebccc2 --- /dev/null +++ b/src/Disks/DiskLocalCheckThread.cpp @@ -0,0 +1,70 @@ +#include + +#include +#include +#include + +namespace DB +{ +static const auto DISK_CHECK_ERROR_SLEEP_MS = 1000; +static const auto DISK_CHECK_ERROR_RETRY_TIME = 3; + +DiskLocalCheckThread::DiskLocalCheckThread(DiskLocal * disk_, ContextPtr context_, UInt64 local_disk_check_period_ms) + : WithContext(context_) + , disk(std::move(disk_)) + , check_period_ms(local_disk_check_period_ms) + , log(&Poco::Logger::get(fmt::format("DiskLocalCheckThread({})", disk->getName()))) +{ + task = getContext()->getSchedulePool().createTask(log->name(), [this] { run(); }); +} + +void DiskLocalCheckThread::startup() +{ + need_stop = false; + retry = 0; + task->activateAndSchedule(); +} + +void DiskLocalCheckThread::run() +{ + if (need_stop) + return; + + bool can_read = disk->canRead(); + bool can_write = disk->canWrite(); + if (can_read) + { + if (disk->broken) + LOG_INFO(log, "Disk {0} seems to be fine. It can be recovered using `SYSTEM RESTART DISK {0}`", disk->getName()); + retry = 0; + if (can_write) + disk->readonly = false; + else + { + disk->readonly = true; + LOG_INFO(log, "Disk {} is readonly", disk->getName()); + } + task->scheduleAfter(check_period_ms); + } + else if (!disk->broken && retry < DISK_CHECK_ERROR_RETRY_TIME) + { + ++retry; + task->scheduleAfter(DISK_CHECK_ERROR_SLEEP_MS); + } + else + { + retry = 0; + disk->broken = true; + LOG_INFO(log, "Disk {} is broken", disk->getName()); + task->scheduleAfter(check_period_ms); + } +} + +void DiskLocalCheckThread::shutdown() +{ + need_stop = true; + task->deactivate(); + LOG_TRACE(log, "DiskLocalCheck thread finished"); +} + +} diff --git a/src/Disks/DiskLocalCheckThread.h b/src/Disks/DiskLocalCheckThread.h new file mode 100644 index 00000000000..eb688d599ca --- /dev/null +++ b/src/Disks/DiskLocalCheckThread.h @@ -0,0 +1,39 @@ +#pragma once + +#include +#include + +namespace Poco +{ +class Logger; +} + +namespace DB +{ +class DiskLocal; + +class DiskLocalCheckThread : WithContext +{ +public: + friend class DiskLocal; + + DiskLocalCheckThread(DiskLocal * disk_, ContextPtr context_, UInt64 local_disk_check_period_ms); + + void startup(); + + void shutdown(); + +private: + bool check(); + void run(); + + DiskLocal * disk; + size_t check_period_ms; + Poco::Logger * log; + std::atomic need_stop{false}; + + BackgroundSchedulePool::TaskHolder task; + size_t retry{}; +}; + +} diff --git a/src/Disks/DiskRestartProxy.cpp b/src/Disks/DiskRestartProxy.cpp index 9bd59513040..fe9dd8421b1 100644 --- a/src/Disks/DiskRestartProxy.cpp +++ b/src/Disks/DiskRestartProxy.cpp @@ -234,6 +234,12 @@ void DiskRestartProxy::removeSharedFile(const String & path, bool keep_s3) DiskDecorator::removeSharedFile(path, keep_s3); } +void DiskRestartProxy::removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) +{ + ReadLock lock (mutex); + DiskDecorator::removeSharedFiles(files, keep_in_remote_fs); +} + void DiskRestartProxy::removeSharedRecursive(const String & path, bool keep_s3) { ReadLock lock (mutex); diff --git a/src/Disks/DiskRestartProxy.h b/src/Disks/DiskRestartProxy.h index 3644539e941..30f553f4fe0 100644 --- a/src/Disks/DiskRestartProxy.h +++ b/src/Disks/DiskRestartProxy.h @@ -54,6 +54,7 @@ public: void removeDirectory(const String & path) override; void removeRecursive(const String & path) override; void removeSharedFile(const String & path, bool keep_s3) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) override; void removeSharedRecursive(const String & path, bool keep_s3) override; void setLastModified(const String & path, const Poco::Timestamp & timestamp) override; Poco::Timestamp getLastModified(const String & path) override; diff --git a/src/Disks/DiskSelector.cpp b/src/Disks/DiskSelector.cpp index 5c117c1b69d..4c80b128b4b 100644 --- a/src/Disks/DiskSelector.cpp +++ b/src/Disks/DiskSelector.cpp @@ -40,7 +40,12 @@ DiskSelector::DiskSelector(const Poco::Util::AbstractConfiguration & config, con disks.emplace(disk_name, factory.create(disk_name, config, disk_config_prefix, context, disks)); } if (!has_default_disk) - disks.emplace(default_disk_name, std::make_shared(default_disk_name, context->getPath(), 0)); + { + disks.emplace( + default_disk_name, + std::make_shared( + default_disk_name, context->getPath(), 0, context, config.getUInt("local_disk_check_period_ms", 0))); + } } @@ -96,7 +101,7 @@ DiskSelectorPtr DiskSelector::updateFromConfig( } writeString(" disappeared from configuration, this change will be applied after restart of ClickHouse", warning); - LOG_WARNING(&Poco::Logger::get("DiskSelector"), warning.str()); + LOG_WARNING(&Poco::Logger::get("DiskSelector"), fmt::runtime(warning.str())); } return result; diff --git a/src/Disks/DiskSelector.h b/src/Disks/DiskSelector.h index 54752215441..0cd1267c6ef 100644 --- a/src/Disks/DiskSelector.h +++ b/src/Disks/DiskSelector.h @@ -37,6 +37,12 @@ public: disks.emplace(name, disk); } + void shutdown() + { + for (auto & e : disks) + e.second->shutdown(); + } + private: DisksMap disks; }; diff --git a/src/Disks/HDFS/DiskHDFS.cpp b/src/Disks/HDFS/DiskHDFS.cpp index 572c908768b..c196276eecb 100644 --- a/src/Disks/HDFS/DiskHDFS.cpp +++ b/src/Disks/HDFS/DiskHDFS.cpp @@ -1,4 +1,7 @@ #include + +#if USE_HDFS + #include #include @@ -73,7 +76,7 @@ DiskHDFS::DiskHDFS( std::unique_ptr DiskHDFS::readFile(const String & path, const ReadSettings & read_settings, std::optional, std::optional) const { - auto metadata = readMeta(path); + auto metadata = readMetadata(path); LOG_TEST(log, "Read from file by path: {}. Existing HDFS objects: {}", @@ -87,8 +90,6 @@ std::unique_ptr DiskHDFS::readFile(const String & path, std::unique_ptr DiskHDFS::writeFile(const String & path, size_t buf_size, WriteMode mode) { - auto metadata = readOrCreateMetaForWriting(path, mode); - /// Path to store new HDFS object. auto file_name = getRandomName(); auto hdfs_path = remote_fs_root_path + file_name; @@ -100,10 +101,13 @@ std::unique_ptr DiskHDFS::writeFile(const String & path auto hdfs_buffer = std::make_unique(hdfs_path, config, settings->replication, buf_size, mode == WriteMode::Rewrite ? O_WRONLY : O_WRONLY | O_APPEND); + auto create_metadata_callback = [this, path, mode, file_name] (size_t count) + { + readOrCreateUpdateAndStoreMetadata(path, mode, false, [file_name, count] (Metadata & metadata) { metadata.addObject(file_name, count); return true; }); + }; - return std::make_unique>(std::move(hdfs_buffer), - std::move(metadata), - file_name); + return std::make_unique>( + std::move(hdfs_buffer), std::move(create_metadata_callback), path); } @@ -179,3 +183,4 @@ void registerDiskHDFS(DiskFactory & factory) } } +#endif diff --git a/src/Disks/HDFS/DiskHDFS.h b/src/Disks/HDFS/DiskHDFS.h index de373d8d6ee..23a108507b4 100644 --- a/src/Disks/HDFS/DiskHDFS.h +++ b/src/Disks/HDFS/DiskHDFS.h @@ -1,5 +1,9 @@ #pragma once +#include + +#if USE_HDFS + #include #include #include @@ -79,3 +83,4 @@ private: }; } +#endif diff --git a/src/Disks/IDisk.cpp b/src/Disks/IDisk.cpp index b1d7b33fec3..42d5f5fce10 100644 --- a/src/Disks/IDisk.cpp +++ b/src/Disks/IDisk.cpp @@ -86,28 +86,4 @@ SyncGuardPtr IDisk::getDirectorySyncGuard(const String & /* path */) const return nullptr; } -std::unique_ptr IDisk::readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const -{ - LOG_TRACE(&Poco::Logger::get("IDisk"), "Read local metafile: {}", path); - return readFile(path, settings, size); -} - -std::unique_ptr IDisk::writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode) -{ - LOG_TRACE(&Poco::Logger::get("IDisk"), "Write local metafile: {}", path); - return writeFile(path, buf_size, mode); -} - -void IDisk::removeMetaFileIfExists(const String & path) -{ - LOG_TRACE(&Poco::Logger::get("IDisk"), "Remove local metafile: {}", path); - removeFileIfExists(path); -} - } diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 665a35459c7..5068ac5dde9 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -197,6 +197,32 @@ public: /// Second bool param is a flag to remove (true) or keep (false) shared data on S3 virtual void removeSharedFileIfExists(const String & path, bool) { removeFileIfExists(path); } + struct RemoveRequest + { + String path; + bool if_exists = false; + + explicit RemoveRequest(String path_, bool if_exists_ = false) + : path(std::move(path_)), if_exists(std::move(if_exists_)) + { + } + }; + + using RemoveBatchRequest = std::vector; + + /// Batch request to remove multiple files. + /// May be much faster for blob storage. + virtual void removeSharedFiles(const RemoveBatchRequest & files, bool keep_in_remote_fs) + { + for (const auto & file : files) + { + if (file.if_exists) + removeSharedFileIfExists(file.path, keep_in_remote_fs); + else + removeSharedFile(file.path, keep_in_remote_fs); + } + } + /// Set last modified time to file or directory at `path`. virtual void setLastModified(const String & path, const Poco::Timestamp & timestamp) = 0; @@ -224,6 +250,9 @@ public: virtual bool isReadOnly() const { return false; } + /// Check if disk is broken. Broken disks will have 0 space and not be used. + virtual bool isBroken() const { return false; } + /// Invoked when Global Context is shutdown. virtual void shutdown() {} @@ -248,28 +277,34 @@ public: /// Applies new settings for disk in runtime. virtual void applyNewSettings(const Poco::Util::AbstractConfiguration &, ContextPtr, const String &, const DisksMap &) {} - /// Open the local file for read and return ReadBufferFromFileBase object. - /// Overridden in IDiskRemote. - /// Used for work with custom metadata. - virtual std::unique_ptr readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const; + /// Quite leaky abstraction. Some disks can use additional disk to store + /// some parts of metadata. In general case we have only one disk itself and + /// return pointer to it. + /// + /// Actually it's a part of IDiskRemote implementation but we have so + /// complex hierarchy of disks (with decorators), so we cannot even + /// dynamic_cast some pointer to IDisk to pointer to IDiskRemote. + virtual std::shared_ptr getMetadataDiskIfExistsOrSelf() { return std::static_pointer_cast(shared_from_this()); } - /// Open the local file for write and return WriteBufferFromFileBase object. - /// Overridden in IDiskRemote. - /// Used for work with custom metadata. - virtual std::unique_ptr writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode); - - virtual void removeMetaFileIfExists(const String & path); + /// Very similar case as for getMetadataDiskIfExistsOrSelf(). If disk has "metadata" + /// it will return mapping for each required path: path -> metadata as string. + /// Only for IDiskRemote. + virtual std::unordered_map getSerializedMetadata(const std::vector & /* paths */) const { return {}; } /// Return reference count for remote FS. - /// Overridden in IDiskRemote. + /// You can ask -- why we have zero and what does it mean? For some unknown reason + /// the decision was made to take 0 as "no references exist", but only file itself left. + /// With normal file system we will get 1 in this case: + /// $ stat clickhouse + /// File: clickhouse + /// Size: 3014014920 Blocks: 5886760 IO Block: 4096 regular file + /// Device: 10301h/66305d Inode: 3109907 Links: 1 + /// Why we have always zero by default? Because normal filesystem + /// manages hardlinks by itself. So you can always remove hardlink and all + /// other alive harlinks will not be removed. virtual UInt32 getRefCount(const String &) const { return 0; } + protected: friend class DiskDecorator; diff --git a/src/Disks/IDiskRemote.cpp b/src/Disks/IDiskRemote.cpp index 706f0f84f32..2a9aded039b 100644 --- a/src/Disks/IDiskRemote.cpp +++ b/src/Disks/IDiskRemote.cpp @@ -24,23 +24,64 @@ namespace ErrorCodes extern const int UNKNOWN_FORMAT; extern const int FILE_ALREADY_EXISTS; extern const int PATH_ACCESS_DENIED;; - extern const int CANNOT_DELETE_DIRECTORY; + extern const int FILE_DOESNT_EXIST; + extern const int BAD_FILE_TYPE; } -/// Load metadata by path or create empty if `create` flag is set. -IDiskRemote::Metadata::Metadata( - const String & remote_fs_root_path_, - DiskPtr metadata_disk_, - const String & metadata_file_path_, - bool create) - : RemoteMetadata(remote_fs_root_path_, metadata_file_path_) - , metadata_disk(metadata_disk_) - , total_size(0), ref_count(0) +IDiskRemote::Metadata IDiskRemote::Metadata::readMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_) { - if (create) - return; + Metadata result(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + result.load(); + return result; +} + + +IDiskRemote::Metadata IDiskRemote::Metadata::createAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync) +{ + Metadata result(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + result.save(sync); + return result; +} + + +IDiskRemote::Metadata IDiskRemote::Metadata::readUpdateAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, IDiskRemote::MetadataUpdater updater) +{ + Metadata result(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + result.load(); + if (updater(result)) + result.save(sync); + return result; +} + + +IDiskRemote::Metadata IDiskRemote::Metadata::createUpdateAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, IDiskRemote::MetadataUpdater updater) +{ + Metadata result(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + updater(result); + result.save(sync); + return result; +} + + +IDiskRemote::Metadata IDiskRemote::Metadata::createAndStoreMetadataIfNotExists(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, bool overwrite) +{ + if (overwrite || !metadata_disk_->exists(metadata_file_path_)) + { + return createAndStoreMetadata(remote_fs_root_path_, metadata_disk_, metadata_file_path_, sync); + } + else + { + auto result = readMetadata(remote_fs_root_path_, metadata_disk_, metadata_file_path_); + if (result.read_only) + throw Exception("File is read-only: " + metadata_file_path_, ErrorCodes::PATH_ACCESS_DENIED); + return result; + } +} + +void IDiskRemote::Metadata::load() +{ try { const ReadSettings read_settings; @@ -102,103 +143,158 @@ IDiskRemote::Metadata::Metadata( } } +/// Load metadata by path or create empty if `create` flag is set. +IDiskRemote::Metadata::Metadata( + const String & remote_fs_root_path_, + DiskPtr metadata_disk_, + const String & metadata_file_path_) + : RemoteMetadata(remote_fs_root_path_, metadata_file_path_) + , metadata_disk(metadata_disk_) + , total_size(0), ref_count(0) +{ +} + void IDiskRemote::Metadata::addObject(const String & path, size_t size) { total_size += size; remote_fs_objects.emplace_back(path, size); } + +void IDiskRemote::Metadata::saveToBuffer(WriteBuffer & buf, bool sync) +{ + writeIntText(VERSION_RELATIVE_PATHS, buf); + writeChar('\n', buf); + + writeIntText(remote_fs_objects.size(), buf); + writeChar('\t', buf); + writeIntText(total_size, buf); + writeChar('\n', buf); + + for (const auto & [remote_fs_object_path, remote_fs_object_size] : remote_fs_objects) + { + writeIntText(remote_fs_object_size, buf); + writeChar('\t', buf); + writeEscapedString(remote_fs_object_path, buf); + writeChar('\n', buf); + } + + writeIntText(ref_count, buf); + writeChar('\n', buf); + + writeBoolText(read_only, buf); + writeChar('\n', buf); + + buf.finalize(); + if (sync) + buf.sync(); + +} + /// Fsync metadata file if 'sync' flag is set. void IDiskRemote::Metadata::save(bool sync) { auto buf = metadata_disk->writeFile(metadata_file_path, 1024); + saveToBuffer(*buf, sync); +} - writeIntText(VERSION_RELATIVE_PATHS, *buf); - writeChar('\n', *buf); +std::string IDiskRemote::Metadata::serializeToString() +{ + WriteBufferFromOwnString write_buf; + saveToBuffer(write_buf, false); + return write_buf.str(); +} - writeIntText(remote_fs_objects.size(), *buf); - writeChar('\t', *buf); - writeIntText(total_size, *buf); - writeChar('\n', *buf); +IDiskRemote::Metadata IDiskRemote::readMetadataUnlocked(const String & path, std::shared_lock &) const +{ + return Metadata::readMetadata(remote_fs_root_path, metadata_disk, path); +} - for (const auto & [remote_fs_object_path, remote_fs_object_size] : remote_fs_objects) + +IDiskRemote::Metadata IDiskRemote::readMetadata(const String & path) const +{ + std::shared_lock lock(metadata_mutex); + return readMetadataUnlocked(path, lock); +} + +IDiskRemote::Metadata IDiskRemote::readUpdateAndStoreMetadata(const String & path, bool sync, IDiskRemote::MetadataUpdater updater) +{ + std::unique_lock lock(metadata_mutex); + return Metadata::readUpdateAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync, updater); +} + + +IDiskRemote::Metadata IDiskRemote::readOrCreateUpdateAndStoreMetadata(const String & path, WriteMode mode, bool sync, IDiskRemote::MetadataUpdater updater) +{ + if (mode == WriteMode::Rewrite || !metadata_disk->exists(path)) { - writeIntText(remote_fs_object_size, *buf); - writeChar('\t', *buf); - writeEscapedString(remote_fs_object_path, *buf); - writeChar('\n', *buf); + std::unique_lock lock(metadata_mutex); + return Metadata::createUpdateAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync, updater); + } + else + { + return Metadata::readUpdateAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync, updater); + } +} + +IDiskRemote::Metadata IDiskRemote::createAndStoreMetadata(const String & path, bool sync) +{ + return Metadata::createAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync); +} + +IDiskRemote::Metadata IDiskRemote::createUpdateAndStoreMetadata(const String & path, bool sync, IDiskRemote::MetadataUpdater updater) +{ + return Metadata::createUpdateAndStoreMetadata(remote_fs_root_path, metadata_disk, path, sync, updater); +} + + +std::unordered_map IDiskRemote::getSerializedMetadata(const std::vector & file_paths) const +{ + std::unordered_map metadatas; + + std::shared_lock lock(metadata_mutex); + + for (const auto & path : file_paths) + { + IDiskRemote::Metadata metadata = readMetadataUnlocked(path, lock); + metadata.ref_count = 0; + metadatas[path] = metadata.serializeToString(); } - writeIntText(ref_count, *buf); - writeChar('\n', *buf); - - writeBoolText(read_only, *buf); - writeChar('\n', *buf); - - buf->finalize(); - if (sync) - buf->sync(); + return metadatas; } -IDiskRemote::Metadata IDiskRemote::readOrCreateMetaForWriting(const String & path, WriteMode mode) -{ - bool exist = exists(path); - if (exist) - { - auto metadata = readMeta(path); - if (metadata.read_only) - throw Exception("File is read-only: " + path, ErrorCodes::PATH_ACCESS_DENIED); - - if (mode == WriteMode::Rewrite) - removeFile(path); /// Remove for re-write. - else - return metadata; - } - - auto metadata = createMeta(path); - /// Save empty metadata to disk to have ability to get file size while buffer is not finalized. - metadata.save(); - - return metadata; -} - - -IDiskRemote::Metadata IDiskRemote::readMeta(const String & path) const -{ - return Metadata(remote_fs_root_path, metadata_disk, path); -} - - -IDiskRemote::Metadata IDiskRemote::createMeta(const String & path) const -{ - return Metadata(remote_fs_root_path, metadata_disk, path, true); -} - - -void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper) +void IDiskRemote::removeMetadata(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper) { LOG_TRACE(log, "Remove file by path: {}", backQuote(metadata_disk->getPath() + path)); + if (!metadata_disk->exists(path)) + throw Exception(ErrorCodes::FILE_DOESNT_EXIST, "Metadata path '{}' doesn't exist", path); + if (!metadata_disk->isFile(path)) - throw Exception(ErrorCodes::CANNOT_DELETE_DIRECTORY, "Path '{}' is a directory", path); + throw Exception(ErrorCodes::BAD_FILE_TYPE, "Path '{}' is not a regular file", path); try { - auto metadata = readMeta(path); + auto metadata_updater = [fs_paths_keeper, this] (Metadata & metadata) + { + if (metadata.ref_count == 0) + { + for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects) + fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path); + return false; + } + else /// In other case decrement number of references, save metadata and delete hardlink. + { + --metadata.ref_count; + } + return true; + }; + + readUpdateAndStoreMetadata(path, false, metadata_updater); + metadata_disk->removeFile(path); /// If there is no references - delete content from remote FS. - if (metadata.ref_count == 0) - { - metadata_disk->removeFile(path); - for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects) - fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path); - } - else /// In other case decrement number of references, save metadata and delete file. - { - --metadata.ref_count; - metadata.save(); - metadata_disk->removeFile(path); - } } catch (const Exception & e) { @@ -216,18 +312,19 @@ void IDiskRemote::removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths } -void IDiskRemote::removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper) +void IDiskRemote::removeMetadataRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper) { checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks. if (metadata_disk->isFile(path)) { - removeMeta(path, fs_paths_keeper); + removeMetadata(path, fs_paths_keeper); } else { - for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) - removeMetaRecursive(it->path(), fs_paths_keeper); + for (auto it = iterateDirectory(path); it->isValid(); it->next()) + removeMetadataRecursive(it->path(), fs_paths_keeper); + metadata_disk->removeDirectory(path); } } @@ -305,16 +402,13 @@ bool IDiskRemote::isFile(const String & path) const void IDiskRemote::createFile(const String & path) { - /// Create empty metadata file. - auto metadata = createMeta(path); - metadata.save(); + createAndStoreMetadata(path, false); } size_t IDiskRemote::getFileSize(const String & path) const { - auto metadata = readMeta(path); - return metadata.total_size; + return readMetadata(path).total_size; } @@ -341,32 +435,45 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path) } -void IDiskRemote::removeSharedFile(const String & path, bool keep_in_remote_fs) +void IDiskRemote::removeSharedFile(const String & path, bool delete_metadata_only) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); - removeMeta(path, fs_paths_keeper); - if (!keep_in_remote_fs) + removeMetadata(path, fs_paths_keeper); + if (!delete_metadata_only) removeFromRemoteFS(fs_paths_keeper); } -void IDiskRemote::removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) +void IDiskRemote::removeSharedFileIfExists(const String & path, bool delete_metadata_only) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); if (metadata_disk->exists(path)) { - removeMeta(path, fs_paths_keeper); - if (!keep_in_remote_fs) + removeMetadata(path, fs_paths_keeper); + if (!delete_metadata_only) removeFromRemoteFS(fs_paths_keeper); } } - -void IDiskRemote::removeSharedRecursive(const String & path, bool keep_in_remote_fs) +void IDiskRemote::removeSharedFiles(const RemoveBatchRequest & files, bool delete_metadata_only) { RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); - removeMetaRecursive(path, fs_paths_keeper); - if (!keep_in_remote_fs) + for (const auto & file : files) + { + bool skip = file.if_exists && !metadata_disk->exists(file.path); + if (!skip) + removeMetadata(file.path, fs_paths_keeper); + } + + if (!delete_metadata_only) + removeFromRemoteFS(fs_paths_keeper); +} + +void IDiskRemote::removeSharedRecursive(const String & path, bool delete_metadata_only) +{ + RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper(); + removeMetadataRecursive(path, fs_paths_keeper); + if (!delete_metadata_only) removeFromRemoteFS(fs_paths_keeper); } @@ -375,9 +482,7 @@ void IDiskRemote::setReadOnly(const String & path) { /// We should store read only flag inside metadata file (instead of using FS flag), /// because we modify metadata file when create hard-links from it. - auto metadata = readMeta(path); - metadata.read_only = true; - metadata.save(); + readUpdateAndStoreMetadata(path, false, [] (Metadata & metadata) { metadata.read_only = true; return true; }); } @@ -401,7 +506,7 @@ void IDiskRemote::createDirectories(const String & path) void IDiskRemote::clearDirectory(const String & path) { - for (auto it{iterateDirectory(path)}; it->isValid(); it->next()) + for (auto it = iterateDirectory(path); it->isValid(); it->next()) if (isFile(it->path())) removeFile(it->path()); } @@ -440,10 +545,7 @@ Poco::Timestamp IDiskRemote::getLastModified(const String & path) void IDiskRemote::createHardLink(const String & src_path, const String & dst_path) { - /// Increment number of references. - auto src = readMeta(src_path); - ++src.ref_count; - src.save(); + readUpdateAndStoreMetadata(src_path, false, [] (Metadata & metadata) { metadata.ref_count++; return true; }); /// Create FS hardlink to metadata file. metadata_disk->createHardLink(src_path, dst_path); @@ -485,7 +587,7 @@ bool IDiskRemote::tryReserve(UInt64 bytes) String IDiskRemote::getUniqueId(const String & path) const { LOG_TRACE(log, "Remote path: {}, Path: {}", remote_fs_root_path, path); - Metadata metadata(remote_fs_root_path, metadata_disk, path); + auto metadata = readMetadata(path); String id; if (!metadata.remote_fs_objects.empty()) id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first; @@ -501,34 +603,17 @@ AsynchronousReaderPtr IDiskRemote::getThreadPoolReader() return reader; } -std::unique_ptr IDiskRemote::readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const -{ - LOG_TRACE(log, "Read metafile: {}", path); - return metadata_disk->readFile(path, settings, size); -} - -std::unique_ptr IDiskRemote::writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode) -{ - LOG_TRACE(log, "Write metafile: {}", path); - return metadata_disk->writeFile(path, buf_size, mode); -} - -void IDiskRemote::removeMetaFileIfExists(const String & path) -{ - LOG_TRACE(log, "Remove metafile: {}", path); - return metadata_disk->removeFileIfExists(path); -} - UInt32 IDiskRemote::getRefCount(const String & path) const { - auto meta = readMeta(path); - return meta.ref_count; + return readMetadata(path).ref_count; +} + +ThreadPool & IDiskRemote::getThreadPoolWriter() +{ + constexpr size_t pool_size = 100; + constexpr size_t queue_size = 1000000; + static ThreadPool writer(pool_size, pool_size, queue_size); + return writer; } } diff --git a/src/Disks/IDiskRemote.h b/src/Disks/IDiskRemote.h index c4f475f5b3e..bdb09804a6c 100644 --- a/src/Disks/IDiskRemote.h +++ b/src/Disks/IDiskRemote.h @@ -6,6 +6,8 @@ #include #include #include +#include +#include #include #include #include @@ -57,16 +59,23 @@ public: size_t thread_pool_size); struct Metadata; + using MetadataUpdater = std::function; const String & getName() const final override { return name; } const String & getPath() const final override { return metadata_disk->getPath(); } - Metadata readMeta(const String & path) const; + /// Methods for working with metadata. For some operations (like hardlink + /// creation) metadata can be updated concurrently from multiple threads + /// (file actually rewritten on disk). So additional RW lock is required for + /// metadata read and write, but not for create new metadata. + Metadata readMetadata(const String & path) const; + Metadata readMetadataUnlocked(const String & path, std::shared_lock &) const; + Metadata readUpdateAndStoreMetadata(const String & path, bool sync, MetadataUpdater updater); + Metadata readOrCreateUpdateAndStoreMetadata(const String & path, WriteMode mode, bool sync, MetadataUpdater updater); - Metadata createMeta(const String & path) const; - - Metadata readOrCreateMetaForWriting(const String & path, WriteMode mode); + Metadata createAndStoreMetadata(const String & path, bool sync); + Metadata createUpdateAndStoreMetadata(const String & path, bool sync, MetadataUpdater updater); UInt64 getTotalSpace() const override { return std::numeric_limits::max(); } @@ -94,11 +103,13 @@ public: void removeRecursive(const String & path) override { removeSharedRecursive(path, false); } - void removeSharedFile(const String & path, bool keep_in_remote_fs) override; + void removeSharedFile(const String & path, bool delete_metadata_only) override; - void removeSharedFileIfExists(const String & path, bool keep_in_remote_fs) override; + void removeSharedFileIfExists(const String & path, bool delete_metadata_only) override; - void removeSharedRecursive(const String & path, bool keep_in_remote_fs) override; + void removeSharedFiles(const RemoveBatchRequest & files, bool delete_metadata_only) override; + + void removeSharedRecursive(const String & path, bool delete_metadata_only) override; void listFiles(const String & path, std::vector & file_names) override; @@ -135,22 +146,16 @@ public: virtual RemoteFSPathKeeperPtr createFSPathKeeper() const = 0; static AsynchronousReaderPtr getThreadPoolReader(); + static ThreadPool & getThreadPoolWriter(); - virtual std::unique_ptr readMetaFile( - const String & path, - const ReadSettings & settings, - std::optional size) const override; - - virtual std::unique_ptr writeMetaFile( - const String & path, - size_t buf_size, - WriteMode mode) override; - - virtual void removeMetaFileIfExists( - const String & path) override; + DiskPtr getMetadataDiskIfExistsOrSelf() override { return metadata_disk; } UInt32 getRefCount(const String & path) const override; + /// Return metadata for each file path. Also, before serialization reset + /// ref_count for each metadata to zero. This function used only for remote + /// fetches/sends in replicated engines. That's why we reset ref_count to zero. + std::unordered_map getSerializedMetadata(const std::vector & file_paths) const override; protected: Poco::Logger * log; const String name; @@ -159,15 +164,16 @@ protected: DiskPtr metadata_disk; private: - void removeMeta(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper); + void removeMetadata(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper); - void removeMetaRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper); + void removeMetadataRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper); bool tryReserve(UInt64 bytes); UInt64 reserved_bytes = 0; UInt64 reservation_count = 0; std::mutex reservation_mutex; + mutable std::shared_mutex metadata_mutex; }; using RemoteDiskPtr = std::shared_ptr; @@ -197,6 +203,7 @@ struct RemoteMetadata struct IDiskRemote::Metadata : RemoteMetadata { + using Updater = std::function; /// Metadata file version. static constexpr UInt32 VERSION_ABSOLUTE_PATHS = 1; static constexpr UInt32 VERSION_RELATIVE_PATHS = 2; @@ -208,22 +215,36 @@ struct IDiskRemote::Metadata : RemoteMetadata size_t total_size = 0; /// Number of references (hardlinks) to this metadata file. + /// + /// FIXME: Why we are tracking it explicetly, without + /// info from filesystem???? UInt32 ref_count = 0; /// Flag indicates that file is read only. bool read_only = false; - /// Load metadata by path or create empty if `create` flag is set. - Metadata(const String & remote_fs_root_path_, - DiskPtr metadata_disk_, - const String & metadata_file_path_, - bool create = false); + Metadata( + const String & remote_fs_root_path_, + DiskPtr metadata_disk_, + const String & metadata_file_path_); void addObject(const String & path, size_t size); + static Metadata readMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_); + static Metadata readUpdateAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, Updater updater); + + static Metadata createAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync); + static Metadata createUpdateAndStoreMetadata(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, Updater updater); + static Metadata createAndStoreMetadataIfNotExists(const String & remote_fs_root_path_, DiskPtr metadata_disk_, const String & metadata_file_path_, bool sync, bool overwrite); + + /// Serialize metadata to string (very same with saveToBuffer) + std::string serializeToString(); + +private: /// Fsync metadata file if 'sync' flag is set. void save(bool sync = false); - + void saveToBuffer(WriteBuffer & buffer, bool sync); + void load(); }; class DiskRemoteReservation final : public IReservation diff --git a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp index c8484e6088d..184fcfe6f8c 100644 --- a/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/AsynchronousReadIndirectBufferFromRemoteFS.cpp @@ -243,7 +243,7 @@ off_t AsynchronousReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence prefetch_future = {}; } - pos = working_buffer.end(); + resetWorkingBuffer(); /** * Lazy ignore. Save number of bytes to ignore and ignore it either for prefetch buffer or current buffer. diff --git a/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp index c21a55d68ac..cbf265ce741 100644 --- a/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/ReadIndirectBufferFromRemoteFS.cpp @@ -64,7 +64,7 @@ off_t ReadIndirectBufferFromRemoteFS::seek(off_t offset_, int whence) throw Exception("Only SEEK_SET or SEEK_CUR modes are allowed.", ErrorCodes::CANNOT_SEEK_THROUGH_FILE); impl->reset(); - pos = working_buffer.end(); + resetWorkingBuffer(); return impl->file_offset_of_buffer_end; } diff --git a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp index 87453440693..f6a80ece3b3 100644 --- a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp +++ b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp @@ -12,15 +12,14 @@ namespace DB template WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS( std::unique_ptr impl_, - IDiskRemote::Metadata metadata_, - const String & remote_fs_path_) + CreateMetadataCallback && create_callback_, + const String & metadata_file_path_) : WriteBufferFromFileDecorator(std::move(impl_)) - , metadata(std::move(metadata_)) - , remote_fs_path(remote_fs_path_) + , create_metadata_callback(std::move(create_callback_)) + , metadata_file_path(metadata_file_path_) { } - template WriteIndirectBufferFromRemoteFS::~WriteIndirectBufferFromRemoteFS() { @@ -34,25 +33,13 @@ WriteIndirectBufferFromRemoteFS::~WriteIndirectBufferFromRemoteFS() } } - template void WriteIndirectBufferFromRemoteFS::finalizeImpl() { WriteBufferFromFileDecorator::finalizeImpl(); - - metadata.addObject(remote_fs_path, count()); - metadata.save(); + create_metadata_callback(count()); } - -template -void WriteIndirectBufferFromRemoteFS::sync() -{ - if (finalized) - metadata.save(true); -} - - #if USE_AWS_S3 template class WriteIndirectBufferFromRemoteFS; diff --git a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h index 8f88eff3b33..54b08ced191 100644 --- a/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h +++ b/src/Disks/IO/WriteIndirectBufferFromRemoteFS.h @@ -9,6 +9,8 @@ namespace DB { +using CreateMetadataCallback = std::function; + /// Stores data in S3/HDFS and adds the object path and object size to metadata file on local FS. template class WriteIndirectBufferFromRemoteFS final : public WriteBufferFromFileDecorator @@ -16,21 +18,18 @@ class WriteIndirectBufferFromRemoteFS final : public WriteBufferFromFileDecorato public: WriteIndirectBufferFromRemoteFS( std::unique_ptr impl_, - IDiskRemote::Metadata metadata_, - const String & remote_fs_path_); + CreateMetadataCallback && create_callback_, + const String & metadata_file_path_); - virtual ~WriteIndirectBufferFromRemoteFS() override; + ~WriteIndirectBufferFromRemoteFS() override; - void sync() override; - - String getFileName() const override { return metadata.metadata_file_path; } + String getFileName() const override { return metadata_file_path; } private: void finalizeImpl() override; - IDiskRemote::Metadata metadata; - - String remote_fs_path; + CreateMetadataCallback create_metadata_callback; + String metadata_file_path; }; } diff --git a/src/Disks/IStoragePolicy.cpp b/src/Disks/IStoragePolicy.cpp new file mode 100644 index 00000000000..2ba6df4be8f --- /dev/null +++ b/src/Disks/IStoragePolicy.cpp @@ -0,0 +1,34 @@ +#include +#include +#include + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int UNKNOWN_VOLUME; + extern const int UNKNOWN_DISK; +} + +DiskPtr IStoragePolicy::getDiskByName(const String & disk_name) const +{ + auto disk = tryGetDiskByName(disk_name); + if (!disk) + throw Exception(ErrorCodes::UNKNOWN_DISK, + "No such disk {} in storage policy {}", backQuote(disk_name), backQuote(getName())); + + return disk; +} + +VolumePtr IStoragePolicy::getVolumeByName(const String & volume_name) const +{ + auto volume = tryGetVolumeByName(volume_name); + if (!volume) + throw Exception(ErrorCodes::UNKNOWN_VOLUME, + "No such volume {} in storage policy {}", backQuote(volume_name), backQuote(getName())); + + return volume; +} + +} diff --git a/src/Disks/IStoragePolicy.h b/src/Disks/IStoragePolicy.h index 8c6d56bdcfd..a5d1120f377 100644 --- a/src/Disks/IStoragePolicy.h +++ b/src/Disks/IStoragePolicy.h @@ -39,7 +39,8 @@ public: /// Used when it's not important, for example for /// mutations files virtual DiskPtr getAnyDisk() const = 0; - virtual DiskPtr getDiskByName(const String & disk_name) const = 0; + virtual DiskPtr tryGetDiskByName(const String & disk_name) const = 0; + DiskPtr getDiskByName(const String & disk_name) const; /// Get free space from most free disk virtual UInt64 getMaxUnreservedFreeSpace() const = 0; /// Reserves space on any volume with index > min_volume_index or returns nullptr @@ -53,7 +54,8 @@ public: virtual ReservationPtr makeEmptyReservationOnLargestDisk() const = 0; /// Get volume by index. virtual VolumePtr getVolume(size_t index) const = 0; - virtual VolumePtr getVolumeByName(const String & volume_name) const = 0; + virtual VolumePtr tryGetVolumeByName(const String & volume_name) const = 0; + VolumePtr getVolumeByName(const String & volume_name) const; /// Checks if storage policy can be replaced by another one. virtual void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const = 0; /// Find volume index, which contains disk diff --git a/src/Disks/IVolume.h b/src/Disks/IVolume.h index d26ddea787d..843ecc8f5e0 100644 --- a/src/Disks/IVolume.h +++ b/src/Disks/IVolume.h @@ -60,6 +60,7 @@ public: DiskPtr getDisk() const { return getDisk(0); } virtual DiskPtr getDisk(size_t i) const { return disks[i]; } + Disks & getDisks() { return disks; } const Disks & getDisks() const { return disks; } /// Returns effective value of whether merges are allowed on this volume (true) or not (false). diff --git a/src/Disks/S3/DiskS3.cpp b/src/Disks/S3/DiskS3.cpp index ed960528abe..8aa4f8f0dff 100644 --- a/src/Disks/S3/DiskS3.cpp +++ b/src/Disks/S3/DiskS3.cpp @@ -9,6 +9,7 @@ #include +#include #include #include @@ -217,7 +218,7 @@ void DiskS3::moveFile(const String & from_path, const String & to_path, bool sen std::unique_ptr DiskS3::readFile(const String & path, const ReadSettings & read_settings, std::optional, std::optional) const { auto settings = current_settings.get(); - auto metadata = readMeta(path); + auto metadata = readMetadata(path); LOG_TEST(log, "Read from file by path: {}. Existing S3 objects: {}", backQuote(metadata_disk->getPath() + path), metadata.remote_fs_objects.size()); @@ -244,10 +245,9 @@ std::unique_ptr DiskS3::readFile(const String & path, co std::unique_ptr DiskS3::writeFile(const String & path, size_t buf_size, WriteMode mode) { auto settings = current_settings.get(); - auto metadata = readOrCreateMetaForWriting(path, mode); /// Path to store new S3 object. - auto s3_path = getRandomASCIIString(); + auto blob_name = getRandomASCIIString(); std::optional object_metadata; if (settings->send_metadata) @@ -256,22 +256,45 @@ std::unique_ptr DiskS3::writeFile(const String & path, object_metadata = { {"path", path} }; - s3_path = "r" + revisionToString(revision) + "-file-" + s3_path; + blob_name = "r" + revisionToString(revision) + "-file-" + blob_name; } LOG_TRACE(log, "{} to file by path: {}. S3 path: {}", - mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + s3_path); + mode == WriteMode::Rewrite ? "Write" : "Append", backQuote(metadata_disk->getPath() + path), remote_fs_root_path + blob_name); + + /// FIXME -- thread pool lead to obscure segfaults + /// ScheduleFunc schedule = [pool = &getThreadPoolWriter(), thread_group = CurrentThread::getGroup()](auto callback) + /// { + /// pool->scheduleOrThrow([callback = std::move(callback), thread_group]() + /// { + /// if (thread_group) + /// CurrentThread::attachTo(thread_group); + + /// SCOPE_EXIT_SAFE( + /// if (thread_group) + /// CurrentThread::detachQueryIfNotDetached(); + /// ); + /// callback(); + /// }); + /// }; auto s3_buffer = std::make_unique( settings->client, bucket, - metadata.remote_fs_root_path + s3_path, + remote_fs_root_path + blob_name, settings->s3_min_upload_part_size, + settings->s3_upload_part_size_multiply_factor, + settings->s3_upload_part_size_multiply_parts_count_threshold, settings->s3_max_single_part_upload_size, std::move(object_metadata), - buf_size); + buf_size /*, std::move(schedule) */); - return std::make_unique>(std::move(s3_buffer), std::move(metadata), s3_path); + auto create_metadata_callback = [this, path, blob_name, mode] (size_t count) + { + readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_name, count] (Metadata & metadata) { metadata.addObject(blob_name, count); return true; }); + }; + + return std::make_unique>(std::move(s3_buffer), std::move(create_metadata_callback), path); } void DiskS3::createHardLink(const String & src_path, const String & dst_path) @@ -293,13 +316,7 @@ void DiskS3::createHardLink(const String & src_path, const String & dst_path, bo createFileOperationObject("hardlink", revision, object_metadata); } - /// Increment number of references. - auto src = readMeta(src_path); - ++src.ref_count; - src.save(); - - /// Create FS hardlink to metadata file. - metadata_disk->createHardLink(src_path, dst_path); + IDiskRemote::createHardLink(src_path, dst_path); } void DiskS3::shutdown() @@ -321,6 +338,8 @@ void DiskS3::createFileOperationObject(const String & operation_name, UInt64 rev bucket, remote_fs_root_path + key, settings->s3_min_upload_part_size, + settings->s3_upload_part_size_multiply_factor, + settings->s3_upload_part_size_multiply_parts_count_threshold, settings->s3_max_single_part_upload_size, metadata); @@ -400,6 +419,8 @@ void DiskS3::saveSchemaVersion(const int & version) bucket, remote_fs_root_path + SCHEMA_VERSION_OBJECT, settings->s3_min_upload_part_size, + settings->s3_upload_part_size_multiply_factor, + settings->s3_upload_part_size_multiply_parts_count_threshold, settings->s3_max_single_part_upload_size); writeIntText(version, buffer); @@ -415,7 +436,7 @@ void DiskS3::migrateFileToRestorableSchema(const String & path) { LOG_TRACE(log, "Migrate file {} to restorable schema", metadata_disk->getPath() + path); - auto meta = readMeta(path); + auto meta = readMetadata(path); for (const auto & [key, _] : meta.remote_fs_objects) { @@ -871,15 +892,19 @@ void DiskS3::processRestoreFiles(const String & source_bucket, const String & so const auto & path = path_entry->second; createDirectories(directoryPath(path)); - auto metadata = createMeta(path); auto relative_key = shrinkKey(source_path, key); /// Copy object if we restore to different bucket / path. if (bucket != source_bucket || remote_fs_root_path != source_path) copyObject(source_bucket, key, bucket, remote_fs_root_path + relative_key, head_result); - metadata.addObject(relative_key, head_result.GetContentLength()); - metadata.save(); + auto updater = [relative_key, head_result] (Metadata & metadata) + { + metadata.addObject(relative_key, head_result.GetContentLength()); + return true; + }; + + createUpdateAndStoreMetadata(path, false, updater); LOG_TRACE(log, "Restored file {}", path); } @@ -1059,6 +1084,8 @@ DiskS3Settings::DiskS3Settings( const std::shared_ptr & client_, size_t s3_max_single_read_retries_, size_t s3_min_upload_part_size_, + size_t s3_upload_part_size_multiply_factor_, + size_t s3_upload_part_size_multiply_parts_count_threshold_, size_t s3_max_single_part_upload_size_, size_t min_bytes_for_seek_, bool send_metadata_, @@ -1068,6 +1095,8 @@ DiskS3Settings::DiskS3Settings( : client(client_) , s3_max_single_read_retries(s3_max_single_read_retries_) , s3_min_upload_part_size(s3_min_upload_part_size_) + , s3_upload_part_size_multiply_factor(s3_upload_part_size_multiply_factor_) + , s3_upload_part_size_multiply_parts_count_threshold(s3_upload_part_size_multiply_parts_count_threshold_) , s3_max_single_part_upload_size(s3_max_single_part_upload_size_) , min_bytes_for_seek(min_bytes_for_seek_) , send_metadata(send_metadata_) diff --git a/src/Disks/S3/DiskS3.h b/src/Disks/S3/DiskS3.h index c5d0722c6c2..698fa6173c2 100644 --- a/src/Disks/S3/DiskS3.h +++ b/src/Disks/S3/DiskS3.h @@ -29,6 +29,8 @@ struct DiskS3Settings const std::shared_ptr & client_, size_t s3_max_single_read_retries_, size_t s3_min_upload_part_size_, + size_t s3_upload_part_size_multiply_factor_, + size_t s3_upload_part_size_multiply_parts_count_threshold_, size_t s3_max_single_part_upload_size_, size_t min_bytes_for_seek_, bool send_metadata_, @@ -39,6 +41,8 @@ struct DiskS3Settings std::shared_ptr client; size_t s3_max_single_read_retries; size_t s3_min_upload_part_size; + size_t s3_upload_part_size_multiply_factor; + size_t s3_upload_part_size_multiply_parts_count_threshold; size_t s3_max_single_part_upload_size; size_t min_bytes_for_seek; bool send_metadata; diff --git a/src/Disks/S3/registerDiskS3.cpp b/src/Disks/S3/registerDiskS3.cpp index f6824a1b3af..9b2e7137d53 100644 --- a/src/Disks/S3/registerDiskS3.cpp +++ b/src/Disks/S3/registerDiskS3.cpp @@ -155,6 +155,8 @@ std::unique_ptr getSettings(const Poco::Util::AbstractConfigurat getClient(config, config_prefix, context), config.getUInt64(config_prefix + ".s3_max_single_read_retries", context->getSettingsRef().s3_max_single_read_retries), config.getUInt64(config_prefix + ".s3_min_upload_part_size", context->getSettingsRef().s3_min_upload_part_size), + config.getUInt64(config_prefix + ".s3_upload_part_size_multiply_factor", context->getSettingsRef().s3_upload_part_size_multiply_factor), + config.getUInt64(config_prefix + ".s3_upload_part_size_multiply_parts_count_threshold", context->getSettingsRef().s3_upload_part_size_multiply_parts_count_threshold), config.getUInt64(config_prefix + ".s3_max_single_part_upload_size", context->getSettingsRef().s3_max_single_part_upload_size), config.getUInt64(config_prefix + ".min_bytes_for_seek", 1024 * 1024), config.getBool(config_prefix + ".send_metadata", false), @@ -174,6 +176,10 @@ void registerDiskS3(DiskFactory & factory) ContextPtr context, const DisksMap & /*map*/) -> DiskPtr { S3::URI uri(Poco::URI(config.getString(config_prefix + ".endpoint"))); + + if (uri.key.empty()) + throw Exception("Empty S3 path specified in disk configuration", ErrorCodes::BAD_ARGUMENTS); + if (uri.key.back() != '/') throw Exception("S3 path must ends with '/', but '" + uri.key + "' doesn't.", ErrorCodes::BAD_ARGUMENTS); @@ -198,7 +204,16 @@ void registerDiskS3(DiskFactory & factory) s3disk->startup(); - if (config.getBool(config_prefix + ".cache_enabled", true)) + +#ifdef NDEBUG + bool use_cache = true; +#else + /// Current S3 cache implementation lead to allocations in destructor of + /// read buffer. + bool use_cache = false; +#endif + + if (config.getBool(config_prefix + ".cache_enabled", use_cache)) { String cache_path = config.getString(config_prefix + ".cache_path", context->getPath() + "disks/" + name + "/cache/"); s3disk = wrapWithCache(s3disk, "s3-cache", cache_path, metadata_path); diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index c47f517bf88..20192c3a29f 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -164,14 +164,22 @@ DiskPtr StoragePolicy::getAnyDisk() const if (volumes.empty()) throw Exception("Storage policy " + backQuote(name) + " has no volumes. It's a bug.", ErrorCodes::LOGICAL_ERROR); - if (volumes[0]->getDisks().empty()) - throw Exception("Volume " + backQuote(name) + "." + backQuote(volumes[0]->getName()) + " has no disks. It's a bug.", ErrorCodes::LOGICAL_ERROR); + for (const auto & volume : volumes) + { + if (volume->getDisks().empty()) + throw Exception("Volume '" + volume->getName() + "' has no disks. It's a bug", ErrorCodes::LOGICAL_ERROR); + for (const auto & disk : volume->getDisks()) + { + if (!disk->isBroken()) + return disk; + } + } - return volumes[0]->getDisks()[0]; + throw Exception(ErrorCodes::NOT_ENOUGH_SPACE, "All disks in storage policy {} are broken", name); } -DiskPtr StoragePolicy::getDiskByName(const String & disk_name) const +DiskPtr StoragePolicy::tryGetDiskByName(const String & disk_name) const { for (auto && volume : volumes) for (auto && disk : volume->getDisks()) @@ -233,6 +241,10 @@ ReservationPtr StoragePolicy::makeEmptyReservationOnLargestDisk() const } } } + if (!max_disk) + throw Exception( + "There is no space on any disk in storage policy: " + name + ". It's likely all disks are broken", + ErrorCodes::NOT_ENOUGH_SPACE); auto reservation = max_disk->reserve(0); if (!reservation) { @@ -253,11 +265,11 @@ VolumePtr StoragePolicy::getVolume(size_t index) const } -VolumePtr StoragePolicy::getVolumeByName(const String & volume_name) const +VolumePtr StoragePolicy::tryGetVolumeByName(const String & volume_name) const { auto it = volume_index_by_volume_name.find(volume_name); if (it == volume_index_by_volume_name.end()) - throw Exception("No such volume " + backQuote(volume_name) + " in storage policy " + backQuote(name), ErrorCodes::UNKNOWN_VOLUME); + return nullptr; return getVolume(it->second); } diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index b09baf09bda..c3f72e01ec8 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -52,7 +52,7 @@ public: /// mutations files DiskPtr getAnyDisk() const override; - DiskPtr getDiskByName(const String & disk_name) const override; + DiskPtr tryGetDiskByName(const String & disk_name) const override; /// Get free space from most free disk UInt64 getMaxUnreservedFreeSpace() const override; @@ -84,7 +84,7 @@ public: /// Get volume by index. VolumePtr getVolume(size_t index) const override; - VolumePtr getVolumeByName(const String & volume_name) const override; + VolumePtr tryGetVolumeByName(const String & volume_name) const override; /// Checks if storage policy can be replaced by another one. void checkCompatibleWith(const StoragePolicyPtr & new_storage_policy) const override; diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index 3f220cbb20a..be565a532bb 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include @@ -127,6 +129,7 @@ FormatSettings getFormatSettings(ContextPtr context, const Settings & settings) format_settings.capn_proto.enum_comparing_mode = settings.format_capn_proto_enum_comparising_mode; format_settings.seekable_read = settings.input_format_allow_seeks; format_settings.msgpack.number_of_columns = settings.input_format_msgpack_number_of_columns; + format_settings.msgpack.output_uuid_representation = settings.output_format_msgpack_uuid_representation; format_settings.max_rows_to_read_for_schema_inference = settings.input_format_max_rows_to_read_for_schema_inference; /// Validate avro_schema_registry_url with RemoteHostFilter when non-empty and in Server context @@ -431,6 +434,9 @@ void FormatFactory::registerFileExtension(const String & extension, const String String FormatFactory::getFormatFromFileName(String file_name, bool throw_if_not_found) { + if (file_name == "stdin") + return getFormatFromFileDescriptor(STDIN_FILENO); + CompressionMethod compression_method = chooseCompressionMethod(file_name, ""); if (CompressionMethod::None != compression_method) { @@ -459,6 +465,25 @@ String FormatFactory::getFormatFromFileName(String file_name, bool throw_if_not_ return it->second; } +String FormatFactory::getFormatFromFileDescriptor(int fd) +{ +#ifdef OS_LINUX + char buf[32] = {'\0'}; + snprintf(buf, sizeof(buf), "/proc/self/fd/%d", fd); + char file_path[PATH_MAX] = {'\0'}; + if (readlink(buf, file_path, sizeof(file_path) - 1) != -1) + return getFormatFromFileName(file_path, false); + return ""; +#elif defined(__APPLE__) + char file_path[PATH_MAX] = {'\0'}; + if (fcntl(fd, F_GETPATH, file_path) != -1) + return getFormatFromFileName(file_path, false); + return ""; +#else + return ""; +#endif +} + void FormatFactory::registerFileSegmentationEngine(const String & name, FileSegmentationEngine file_segmentation_engine) { auto & target = dict[name].file_segmentation_engine; diff --git a/src/Formats/FormatFactory.h b/src/Formats/FormatFactory.h index 228d5234959..344dabd3f4d 100644 --- a/src/Formats/FormatFactory.h +++ b/src/Formats/FormatFactory.h @@ -187,6 +187,7 @@ public: /// Register file extension for format void registerFileExtension(const String & extension, const String & format_name); String getFormatFromFileName(String file_name, bool throw_if_not_found = false); + String getFormatFromFileDescriptor(int fd); /// Register schema readers for format its name. void registerSchemaReader(const String & name, SchemaReaderCreator schema_reader_creator); diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index b484d623944..265c879e768 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -231,9 +231,17 @@ struct FormatSettings EnumComparingMode enum_comparing_mode = EnumComparingMode::BY_VALUES; } capn_proto; + enum class MsgPackUUIDRepresentation + { + STR, // Output UUID as a string of 36 characters. + BIN, // Output UUID as 16-bytes binary. + EXT, // Output UUID as ExtType = 2 + }; + struct { UInt64 number_of_columns = 0; + MsgPackUUIDRepresentation output_uuid_representation = MsgPackUUIDRepresentation::EXT; } msgpack; }; diff --git a/src/Formats/MsgPackExtensionTypes.h b/src/Formats/MsgPackExtensionTypes.h new file mode 100644 index 00000000000..139d2f9047b --- /dev/null +++ b/src/Formats/MsgPackExtensionTypes.h @@ -0,0 +1,11 @@ +#pragma once + +namespace DB +{ + +enum class MsgPackExtensionTypes +{ + UUID = 0x02, +}; + +} diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index b59db12a16c..389d25a1f46 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -36,6 +36,7 @@ # include # include # include +# include # include # include # include @@ -2163,7 +2164,7 @@ namespace for (auto & desc : field_descs_) field_infos.emplace_back(std::move(desc.column_indices), *desc.field_descriptor, std::move(desc.field_serializer)); - std::sort(field_infos.begin(), field_infos.end(), + ::sort(field_infos.begin(), field_infos.end(), [](const FieldInfo & lhs, const FieldInfo & rhs) { return lhs.field_tag < rhs.field_tag; }); for (size_t i : collections::range(field_infos.size())) @@ -2643,7 +2644,7 @@ namespace missing_column_indices.clear(); missing_column_indices.reserve(column_names.size() - used_column_indices.size()); auto used_column_indices_sorted = std::move(used_column_indices); - std::sort(used_column_indices_sorted.begin(), used_column_indices_sorted.end()); + ::sort(used_column_indices_sorted.begin(), used_column_indices_sorted.end()); boost::range::set_difference(collections::range(column_names.size()), used_column_indices_sorted, std::back_inserter(missing_column_indices)); @@ -2755,7 +2756,7 @@ namespace } /// Shorter suffixes first. - std::sort(out_field_descriptors_with_suffixes.begin(), out_field_descriptors_with_suffixes.end(), + ::sort(out_field_descriptors_with_suffixes.begin(), out_field_descriptors_with_suffixes.end(), [](const std::pair & f1, const std::pair & f2) { diff --git a/src/Formats/ReadSchemaUtils.cpp b/src/Formats/ReadSchemaUtils.cpp index 37067eae64f..559fac4cfaa 100644 --- a/src/Formats/ReadSchemaUtils.cpp +++ b/src/Formats/ReadSchemaUtils.cpp @@ -17,7 +17,12 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -ColumnsDescription readSchemaFromFormat(const String & format_name, const std::optional & format_settings, ReadBufferCreator read_buffer_creator, ContextPtr context) +ColumnsDescription readSchemaFromFormat( + const String & format_name, + const std::optional & format_settings, + ReadBufferCreator read_buffer_creator, + ContextPtr context, + std::unique_ptr & buf_out) { NamesAndTypesList names_and_types; if (FormatFactory::instance().checkIfFormatHasExternalSchemaReader(format_name)) @@ -34,11 +39,11 @@ ColumnsDescription readSchemaFromFormat(const String & format_name, const std::o } else if (FormatFactory::instance().checkIfFormatHasSchemaReader(format_name)) { - auto read_buf = read_buffer_creator(); - if (read_buf->eof()) + buf_out = read_buffer_creator(); + if (buf_out->eof()) throw Exception(ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, "Cannot extract table structure from {} format file, file is empty", format_name); - auto schema_reader = FormatFactory::instance().getSchemaReader(format_name, *read_buf, context, format_settings); + auto schema_reader = FormatFactory::instance().getSchemaReader(format_name, *buf_out, context, format_settings); try { names_and_types = schema_reader->readSchema(); @@ -54,6 +59,12 @@ ColumnsDescription readSchemaFromFormat(const String & format_name, const std::o return ColumnsDescription(names_and_types); } +ColumnsDescription readSchemaFromFormat(const String & format_name, const std::optional & format_settings, ReadBufferCreator read_buffer_creator, ContextPtr context) +{ + std::unique_ptr buf_out; + return readSchemaFromFormat(format_name, format_settings, read_buffer_creator, context, buf_out); +} + DataTypePtr generalizeDataType(DataTypePtr type) { WhichDataType which(type); diff --git a/src/Formats/ReadSchemaUtils.h b/src/Formats/ReadSchemaUtils.h index fb43acc3cd6..4446393a581 100644 --- a/src/Formats/ReadSchemaUtils.h +++ b/src/Formats/ReadSchemaUtils.h @@ -15,7 +15,19 @@ namespace DB /// If format doesn't have any schema reader or a schema reader /// couldn't determine the schema, an exception will be thrown. using ReadBufferCreator = std::function()>; -ColumnsDescription readSchemaFromFormat(const String & format_name, const std::optional & format_settings, ReadBufferCreator read_buffer_creator, ContextPtr context); +ColumnsDescription readSchemaFromFormat( + const String & format_name, + const std::optional & format_settings, + ReadBufferCreator read_buffer_creator, + ContextPtr context); + +/// If ReadBuffer is created, it will be written to buf_out. +ColumnsDescription readSchemaFromFormat( + const String & format_name, + const std::optional & format_settings, + ReadBufferCreator read_buffer_creator, + ContextPtr context, + std::unique_ptr & buf_out); /// Convert type to the most general type: /// - IntN, UIntN, FloatN, Decimal -> Float64 diff --git a/src/Functions/FunctionMathUnary.h b/src/Functions/FunctionMathUnary.h index e0b9355e0a6..fa10c004e87 100644 --- a/src/Functions/FunctionMathUnary.h +++ b/src/Functions/FunctionMathUnary.h @@ -125,7 +125,7 @@ private: { const auto & src_data = col->getData(); const size_t size = src_data.size(); - UInt32 scale = src_data.getScale(); + UInt32 scale = col->getScale(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); diff --git a/src/Functions/FunctionUnixTimestamp64.h b/src/Functions/FunctionUnixTimestamp64.h index 5248f524a2b..8c248d79c4b 100644 --- a/src/Functions/FunctionUnixTimestamp64.h +++ b/src/Functions/FunctionUnixTimestamp64.h @@ -18,6 +18,7 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int DECIMAL_OVERFLOW; + extern const int ILLEGAL_COLUMN; } /// Cast DateTime64 to Int64 representation narrowed down (or scaled up) to any scale value defined in Impl. @@ -108,8 +109,8 @@ public: if (arguments.size() < 1 || arguments.size() > 2) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} takes one or two arguments", name); - if (!typeid_cast(arguments[0].type.get())) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The first argument for function {} must be Int64", name); + if (!isInteger(arguments[0].type)) + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "The first argument for function {} must be integer", name); std::string timezone; if (arguments.size() == 2) @@ -118,21 +119,48 @@ public: return std::make_shared(target_scale, timezone); } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + template + bool executeType(auto & result_column, const ColumnsWithTypeAndName & arguments, size_t input_rows_count) const { const auto & src = arguments[0]; const auto & col = *src.column; - auto res_column = ColumnDecimal::create(input_rows_count, target_scale); - auto & result_data = res_column->getData(); + if (!checkAndGetColumn>(col)) + return 0; - const auto & source_data = typeid_cast(col).getData(); + auto & result_data = result_column->getData(); + + const auto & source_data = typeid_cast &>(col).getData(); for (size_t i = 0; i < input_rows_count; ++i) result_data[i] = source_data[i]; - return res_column; + return 1; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + auto result_column = ColumnDecimal::create(input_rows_count, target_scale); + + if (!((executeType(result_column, arguments, input_rows_count)) + || (executeType(result_column, arguments, input_rows_count)) + || (executeType(result_column, arguments, input_rows_count)) + || (executeType(result_column, arguments, input_rows_count)) + || (executeType(result_column, arguments, input_rows_count)) + || (executeType(result_column, arguments, input_rows_count)) + || (executeType(result_column, arguments, input_rows_count)) + || (executeType(result_column, arguments, input_rows_count)) + || (executeType(result_column, arguments, input_rows_count)))) + { + throw Exception(ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of first argument of function {}", + arguments[0].column->getName(), + getName()); + } + + return result_column; + } + }; } diff --git a/src/Functions/FunctionsConversion.h b/src/Functions/FunctionsConversion.h index d65d0604547..909803d7cd7 100644 --- a/src/Functions/FunctionsConversion.h +++ b/src/Functions/FunctionsConversion.h @@ -152,9 +152,11 @@ struct ConvertImpl if (const ColVecFrom * col_from = checkAndGetColumn(named_from.column.get())) { typename ColVecTo::MutablePtr col_to = nullptr; + if constexpr (IsDataTypeDecimal) { UInt32 scale; + if constexpr (std::is_same_v || std::is_same_v) { @@ -208,11 +210,11 @@ struct ConvertImpl bool convert_result = false; if constexpr (IsDataTypeDecimal && IsDataTypeDecimal) - convert_result = tryConvertDecimals(vec_from[i], vec_from.getScale(), vec_to.getScale(), result); + convert_result = tryConvertDecimals(vec_from[i], col_from->getScale(), col_to->getScale(), result); else if constexpr (IsDataTypeDecimal && IsDataTypeNumber) - convert_result = tryConvertFromDecimal(vec_from[i], vec_from.getScale(), result); + convert_result = tryConvertFromDecimal(vec_from[i], col_from->getScale(), result); else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) - convert_result = tryConvertToDecimal(vec_from[i], vec_to.getScale(), result); + convert_result = tryConvertToDecimal(vec_from[i], col_to->getScale(), result); if (convert_result) vec_to[i] = result; @@ -225,11 +227,11 @@ struct ConvertImpl else { if constexpr (IsDataTypeDecimal && IsDataTypeDecimal) - vec_to[i] = convertDecimals(vec_from[i], vec_from.getScale(), vec_to.getScale()); + vec_to[i] = convertDecimals(vec_from[i], col_from->getScale(), col_to->getScale()); else if constexpr (IsDataTypeDecimal && IsDataTypeNumber) - vec_to[i] = convertFromDecimal(vec_from[i], vec_from.getScale()); + vec_to[i] = convertFromDecimal(vec_from[i], col_from->getScale()); else if constexpr (IsDataTypeNumber && IsDataTypeDecimal) - vec_to[i] = convertToDecimal(vec_from[i], vec_to.getScale()); + vec_to[i] = convertToDecimal(vec_from[i], col_to->getScale()); else throw Exception("Unsupported data type in conversion function", ErrorCodes::CANNOT_CONVERT_TYPE); } @@ -820,7 +822,7 @@ struct ConvertImpl) data_to.resize(size * (strlen("YYYY-MM-DD hh:mm:ss") + 1)); else if constexpr (std::is_same_v) - data_to.resize(size * (strlen("YYYY-MM-DD hh:mm:ss.") + vec_from.getScale() + 1)); + data_to.resize(size * (strlen("YYYY-MM-DD hh:mm:ss.") + col_from->getScale() + 1)); else data_to.resize(size * 3); /// Arbitrary @@ -1169,7 +1171,7 @@ struct ConvertThroughParsing if constexpr (to_datetime64) { DateTime64 res = 0; - parseDateTime64BestEffort(res, vec_to.getScale(), read_buffer, *local_time_zone, *utc_time_zone); + parseDateTime64BestEffort(res, col_to->getScale(), read_buffer, *local_time_zone, *utc_time_zone); vec_to[i] = res; } else @@ -1184,7 +1186,7 @@ struct ConvertThroughParsing if constexpr (to_datetime64) { DateTime64 res = 0; - parseDateTime64BestEffortUS(res, vec_to.getScale(), read_buffer, *local_time_zone, *utc_time_zone); + parseDateTime64BestEffortUS(res, col_to->getScale(), read_buffer, *local_time_zone, *utc_time_zone); vec_to[i] = res; } else @@ -1199,12 +1201,12 @@ struct ConvertThroughParsing if constexpr (to_datetime64) { DateTime64 value = 0; - readDateTime64Text(value, vec_to.getScale(), read_buffer, *local_time_zone); + readDateTime64Text(value, col_to->getScale(), read_buffer, *local_time_zone); vec_to[i] = value; } else if constexpr (IsDataTypeDecimal) SerializationDecimal::readText( - vec_to[i], read_buffer, ToDataType::maxPrecision(), vec_to.getScale()); + vec_to[i], read_buffer, ToDataType::maxPrecision(), col_to->getScale()); else { parseImpl(vec_to[i], read_buffer, local_time_zone); @@ -1223,7 +1225,7 @@ struct ConvertThroughParsing if constexpr (to_datetime64) { DateTime64 res = 0; - parsed = tryParseDateTime64BestEffort(res, vec_to.getScale(), read_buffer, *local_time_zone, *utc_time_zone); + parsed = tryParseDateTime64BestEffort(res, col_to->getScale(), read_buffer, *local_time_zone, *utc_time_zone); vec_to[i] = res; } else @@ -1244,12 +1246,12 @@ struct ConvertThroughParsing if constexpr (to_datetime64) { DateTime64 value = 0; - parsed = tryReadDateTime64Text(value, vec_to.getScale(), read_buffer, *local_time_zone); + parsed = tryReadDateTime64Text(value, col_to->getScale(), read_buffer, *local_time_zone); vec_to[i] = value; } else if constexpr (IsDataTypeDecimal) parsed = SerializationDecimal::tryReadText( - vec_to[i], read_buffer, ToDataType::maxPrecision(), vec_to.getScale()); + vec_to[i], read_buffer, ToDataType::maxPrecision(), col_to->getScale()); else parsed = tryParseImpl(vec_to[i], read_buffer, local_time_zone); } diff --git a/src/Functions/FunctionsRound.h b/src/Functions/FunctionsRound.h index edba82a5b4e..1a8bf85167f 100644 --- a/src/Functions/FunctionsRound.h +++ b/src/Functions/FunctionsRound.h @@ -17,6 +17,7 @@ #include #include #include +#include #include #ifdef __SSE4_1__ @@ -158,7 +159,6 @@ struct IntegerRoundingComputation switch (scale_mode) { case ScaleMode::Zero: - return x; case ScaleMode::Positive: return x; case ScaleMode::Negative: @@ -170,10 +170,15 @@ struct IntegerRoundingComputation static ALWAYS_INLINE void compute(const T * __restrict in, size_t scale, T * __restrict out) { - if (sizeof(T) <= sizeof(scale) && scale > size_t(std::numeric_limits::max())) - *out = 0; - else - *out = compute(*in, scale); + if constexpr (sizeof(T) <= sizeof(scale) && scale_mode == ScaleMode::Negative) + { + if (scale > size_t(std::numeric_limits::max())) + { + *out = 0; + return; + } + } + *out = compute(*in, scale); } }; @@ -422,9 +427,9 @@ private: using Container = typename ColumnDecimal::Container; public: - static NO_INLINE void apply(const Container & in, Container & out, Scale scale_arg) + static NO_INLINE void apply(const Container & in, UInt32 in_scale, Container & out, Scale scale_arg) { - scale_arg = in.getScale() - scale_arg; + scale_arg = in_scale - scale_arg; if (scale_arg > 0) { size_t scale = intExp10(scale_arg); @@ -498,11 +503,11 @@ public: const auto * const col = checkAndGetColumn>(col_general); const typename ColumnDecimal::Container & vec_src = col->getData(); - auto col_res = ColumnDecimal::create(vec_src.size(), vec_src.getScale()); + auto col_res = ColumnDecimal::create(vec_src.size(), col->getScale()); auto & vec_res = col_res->getData(); if (!vec_res.empty()) - DecimalRoundingImpl::apply(col->getData(), vec_res, scale_arg); + DecimalRoundingImpl::apply(col->getData(), col->getScale(), vec_res, scale_arg); return col_res; } @@ -738,7 +743,7 @@ private: for (size_t i = 0; i < boundaries.size(); ++i) boundary_values[i] = boundaries[i].get(); - std::sort(boundary_values.begin(), boundary_values.end()); + ::sort(boundary_values.begin(), boundary_values.end()); boundary_values.erase(std::unique(boundary_values.begin(), boundary_values.end()), boundary_values.end()); size_t size = src.size(); diff --git a/src/Functions/FunctionsStringArray.h b/src/Functions/FunctionsStringArray.h index a6e705bb1af..b1de017120c 100644 --- a/src/Functions/FunctionsStringArray.h +++ b/src/Functions/FunctionsStringArray.h @@ -26,6 +26,7 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; extern const int ILLEGAL_COLUMN; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } @@ -69,6 +70,8 @@ public: static constexpr auto name = "alphaTokens"; static String getName() { return name; } + static bool isVariadic() { return false; } + static size_t getNumberOfArguments() { return 1; } /// Check the type of the function's arguments. @@ -127,6 +130,7 @@ public: static constexpr auto name = "splitByNonAlpha"; static String getName() { return name; } + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 1; } /// Check the type of the function's arguments. @@ -185,6 +189,7 @@ public: static constexpr auto name = "splitByWhitespace"; static String getName() { return name; } + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 1; } /// Check the type of the function's arguments. @@ -239,14 +244,23 @@ private: Pos end; char sep; + std::optional max_split; + UInt64 curr_split = 0; public: static constexpr auto name = "splitByChar"; static String getName() { return name; } - static size_t getNumberOfArguments() { return 2; } + static bool isVariadic() { return true; } + static size_t getNumberOfArguments() { return 0; } static void checkArguments(const DataTypes & arguments) { + if (arguments.size() < 2 || arguments.size() > 3) + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function '{}' needs at least 2 arguments, at most 3 arguments; passed {}.", + arguments.size()); + if (!isString(arguments[0])) throw Exception("Illegal type " + arguments[0]->getName() + " of first argument of function " + getName() + ". Must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); @@ -254,6 +268,13 @@ public: if (!isString(arguments[1])) throw Exception("Illegal type " + arguments[1]->getName() + " of second argument of function " + getName() + ". Must be String.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + if (arguments.size() == 3 && !isNativeInteger(arguments[2])) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Third argument for function '{}' must be integer, got '{}' instead", + getName(), + arguments[2]->getName()); } void init(const ColumnsWithTypeAndName & arguments) @@ -271,6 +292,39 @@ public: throw Exception("Illegal separator for function " + getName() + ". Must be exactly one byte.", ErrorCodes::BAD_ARGUMENTS); sep = sep_str[0]; + + if (arguments.size() > 2) + { + if (!((max_split = getMaxSplit(arguments[2])) + || (max_split = getMaxSplit(arguments[2])) + || (max_split = getMaxSplit(arguments[2])) + || (max_split = getMaxSplit(arguments[2])) + || (max_split = getMaxSplit(arguments[2])) + || (max_split = getMaxSplit(arguments[2])) + || (max_split = getMaxSplit(arguments[2])) + || (max_split = getMaxSplit(arguments[2])))) + { + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal column {} of third argument of function {}", + arguments[2].column->getName(), + getName()); + } + } + } + + template + std::optional getMaxSplit(const ColumnWithTypeAndName & argument) + { + const auto * col = checkAndGetColumnConst>(argument.column.get()); + if (!col) + return std::nullopt; + + auto value = col->template getValue(); + if (value < 0) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, "Illegal column {} of third argument of function {}", argument.column->getName(), getName()); + return value; } /// Returns the position of the argument, that is the column of strings @@ -291,12 +345,19 @@ public: return false; token_begin = pos; - pos = reinterpret_cast(memchr(pos, sep, end - pos)); + if (unlikely(max_split && curr_split >= *max_split)) + { + token_end = end; + pos = nullptr; + return true; + } + pos = reinterpret_cast(memchr(pos, sep, end - pos)); if (pos) { token_end = pos; ++pos; + ++curr_split; } else token_end = end; @@ -317,6 +378,7 @@ private: public: static constexpr auto name = "splitByString"; static String getName() { return name; } + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 2; } static void checkArguments(const DataTypes & arguments) @@ -394,6 +456,8 @@ private: public: static constexpr auto name = "splitByRegexp"; static String getName() { return name; } + + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 2; } /// Check the type of function arguments. @@ -477,6 +541,7 @@ private: public: static constexpr auto name = "extractAll"; static String getName() { return name; } + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 2; } /// Check the type of function arguments. @@ -556,6 +621,8 @@ public: bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + bool isVariadic() const override { return Generator::isVariadic(); } + size_t getNumberOfArguments() const override { return Generator::getNumberOfArguments(); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override diff --git a/src/Functions/URL/URLHierarchy.cpp b/src/Functions/URL/URLHierarchy.cpp index 58d66d8a49d..e21450847b7 100644 --- a/src/Functions/URL/URLHierarchy.cpp +++ b/src/Functions/URL/URLHierarchy.cpp @@ -20,6 +20,7 @@ public: static constexpr auto name = "URLPathHierarchy"; static String getName() { return name; } + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 1; } static void checkArguments(const DataTypes & arguments) diff --git a/src/Functions/URL/URLPathHierarchy.cpp b/src/Functions/URL/URLPathHierarchy.cpp index e801f920881..6f8832ddf65 100644 --- a/src/Functions/URL/URLPathHierarchy.cpp +++ b/src/Functions/URL/URLPathHierarchy.cpp @@ -19,6 +19,7 @@ public: static constexpr auto name = "URLHierarchy"; static String getName() { return name; } + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 1; } static void checkArguments(const DataTypes & arguments) diff --git a/src/Functions/URL/extractURLParameterNames.cpp b/src/Functions/URL/extractURLParameterNames.cpp index ff0b8c2a035..377e969a6b7 100644 --- a/src/Functions/URL/extractURLParameterNames.cpp +++ b/src/Functions/URL/extractURLParameterNames.cpp @@ -19,6 +19,7 @@ public: static constexpr auto name = "extractURLParameterNames"; static String getName() { return name; } + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 1; } static void checkArguments(const DataTypes & arguments) diff --git a/src/Functions/URL/extractURLParameters.cpp b/src/Functions/URL/extractURLParameters.cpp index 1eb0ae7f735..fb595c23170 100644 --- a/src/Functions/URL/extractURLParameters.cpp +++ b/src/Functions/URL/extractURLParameters.cpp @@ -19,6 +19,7 @@ public: static constexpr auto name = "extractURLParameters"; static String getName() { return name; } + static bool isVariadic() { return false; } static size_t getNumberOfArguments() { return 1; } static void checkArguments(const DataTypes & arguments) diff --git a/src/Functions/addressToLine.cpp b/src/Functions/addressToLine.cpp index c3e48913e97..6c9eba160cf 100644 --- a/src/Functions/addressToLine.cpp +++ b/src/Functions/addressToLine.cpp @@ -1,153 +1,58 @@ #if defined(__ELF__) && !defined(__FreeBSD__) #include -#include -#include -#include #include -#include #include -#include #include #include #include #include -#include -#include -#include -#include +#include namespace DB { -namespace ErrorCodes -{ - extern const int ILLEGAL_COLUMN; - extern const int ILLEGAL_TYPE_OF_ARGUMENT; - extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; -} - namespace { -class FunctionAddressToLine : public IFunction +class FunctionAddressToLine: public FunctionAddressToLineBase { public: static constexpr auto name = "addressToLine"; + String getName() const override { return name; } static FunctionPtr create(ContextPtr context) { context->checkAccess(AccessType::addressToLine); return std::make_shared(); } - - String getName() const override +protected: + DataTypePtr getDataType() const override { - return name; - } - - size_t getNumberOfArguments() const override - { - return 1; - } - - bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - - DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override - { - if (arguments.size() != 1) - throw Exception("Function " + getName() + " needs exactly one argument; passed " - + toString(arguments.size()) + ".", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - - const auto & type = arguments[0].type; - - if (!WhichDataType(type.get()).isUInt64()) - throw Exception("The only argument for function " + getName() + " must be UInt64. Found " - + type->getName() + " instead.", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); - return std::make_shared(); } - - bool useDefaultImplementationForConstants() const override + ColumnPtr getResultColumn(const typename ColumnVector::Container & data, size_t input_rows_count) const override { - return true; - } - - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override - { - const ColumnPtr & column = arguments[0].column; - const ColumnUInt64 * column_concrete = checkAndGetColumn(column.get()); - - if (!column_concrete) - throw Exception("Illegal column " + column->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); - - const typename ColumnVector::Container & data = column_concrete->getData(); auto result_column = ColumnString::create(); - for (size_t i = 0; i < input_rows_count; ++i) { StringRef res_str = implCached(data[i]); result_column->insertData(res_str.data, res_str.size); } - return result_column; } -private: - struct Cache + void setResult(StringRef & result, const Dwarf::LocationInfo & location, const std::vector &) const override { - std::mutex mutex; - Arena arena; - using Map = HashMap; - Map map; - std::unordered_map dwarfs; - }; + const char * arena_begin = nullptr; + WriteBufferFromArena out(cache.arena, arena_begin); - mutable Cache cache; + writeString(location.file.toString(), out); + writeChar(':', out); + writeIntText(location.line, out); - StringRef impl(uintptr_t addr) const - { - auto symbol_index_ptr = SymbolIndex::instance(); - const SymbolIndex & symbol_index = *symbol_index_ptr; - - if (const auto * object = symbol_index.findObject(reinterpret_cast(addr))) - { - auto dwarf_it = cache.dwarfs.try_emplace(object->name, object->elf).first; - if (!std::filesystem::exists(object->name)) - return {}; - - Dwarf::LocationInfo location; - std::vector frames; // NOTE: not used in FAST mode. - if (dwarf_it->second.findAddress(addr - uintptr_t(object->address_begin), location, Dwarf::LocationInfoMode::FAST, frames)) - { - const char * arena_begin = nullptr; - WriteBufferFromArena out(cache.arena, arena_begin); - - writeString(location.file.toString(), out); - writeChar(':', out); - writeIntText(location.line, out); - - return out.complete(); - } - else - { - return object->name; - } - } - else - return {}; - } - - StringRef implCached(uintptr_t addr) const - { - Cache::Map::LookupResult it; - bool inserted; - std::lock_guard lock(cache.mutex); - cache.map.emplace(addr, it, inserted); - if (inserted) - it->getMapped() = impl(addr); - return it->getMapped(); + result = out.complete(); } }; diff --git a/src/Functions/addressToLine.h b/src/Functions/addressToLine.h new file mode 100644 index 00000000000..8216f114b2e --- /dev/null +++ b/src/Functions/addressToLine.h @@ -0,0 +1,133 @@ +#pragma once +#if defined(__ELF__) && !defined(__FreeBSD__) + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int ILLEGAL_COLUMN; + extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +template +class FunctionAddressToLineBase : public IFunction +{ +public: + + size_t getNumberOfArguments() const override { return 1; } + + bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } + + DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override + { + if (arguments.size() != 1) + throw Exception( + "Function " + getName() + " needs exactly one argument; passed " + toString(arguments.size()) + ".", + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + const auto & type = arguments[0].type; + + if (!WhichDataType(type.get()).isUInt64()) + throw Exception( + "The only argument for function " + getName() + " must be UInt64. Found " + type->getName() + " instead.", + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); + + return getDataType(); + } + + bool useDefaultImplementationForConstants() const override { return true; } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override + { + const ColumnPtr & column = arguments[0].column; + const ColumnUInt64 * column_concrete = checkAndGetColumn(column.get()); + + if (!column_concrete) + throw Exception( + "Illegal column " + column->getName() + " of argument of function " + getName(), ErrorCodes::ILLEGAL_COLUMN); + + const typename ColumnVector::Container & data = column_concrete->getData(); + return getResultColumn(data, input_rows_count); + } + +protected: + virtual DataTypePtr getDataType() const = 0; + virtual ColumnPtr getResultColumn(const typename ColumnVector::Container & data, size_t input_rows_count) const = 0; + virtual void + setResult(ResultT & result, const Dwarf::LocationInfo & location, const std::vector & frames) const = 0; + + struct Cache + { + std::mutex mutex; + Arena arena; + using Map = HashMap; + Map map; + std::unordered_map dwarfs; + }; + + mutable Cache cache; + + ResultT impl(uintptr_t addr) const + { + auto symbol_index_ptr = SymbolIndex::instance(); + const SymbolIndex & symbol_index = *symbol_index_ptr; + + if (const auto * object = symbol_index.findObject(reinterpret_cast(addr))) + { + auto dwarf_it = cache.dwarfs.try_emplace(object->name, object->elf).first; + if (!std::filesystem::exists(object->name)) + return {}; + + Dwarf::LocationInfo location; + std::vector frames; // NOTE: not used in FAST mode. + ResultT result; + if (dwarf_it->second.findAddress(addr - uintptr_t(object->address_begin), location, locationInfoMode, frames)) + { + setResult(result, location, frames); + return result; + } + else + return {object->name}; + } + else + return {}; + } + + ResultT implCached(uintptr_t addr) const + { + typename Cache::Map::LookupResult it; + bool inserted; + std::lock_guard lock(cache.mutex); + cache.map.emplace(addr, it, inserted); + if (inserted) + it->getMapped() = impl(addr); + return it->getMapped(); + } +}; + +} + +#endif diff --git a/src/Functions/addressToLineWithInlines.cpp b/src/Functions/addressToLineWithInlines.cpp new file mode 100644 index 00000000000..c3e62bd802e --- /dev/null +++ b/src/Functions/addressToLineWithInlines.cpp @@ -0,0 +1,99 @@ +#if defined(__ELF__) && !defined(__FreeBSD__) + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ + +namespace +{ + +class FunctionAddressToLineWithInlines: public FunctionAddressToLineBase +{ +public: + static constexpr auto name = "addressToLineWithInlines"; + String getName() const override { return name; } + static FunctionPtr create(ContextPtr context) + { + context->checkAccess(AccessType::addressToLineWithInlines); + return std::make_shared(); + } + +protected: + DataTypePtr getDataType() const override + { + return std::make_shared(std::make_shared()); + } + + ColumnPtr getResultColumn(const typename ColumnVector::Container & data, size_t input_rows_count) const override + { + auto result_column = ColumnArray::create(ColumnString::create()); + ColumnString & result_strings = typeid_cast(result_column->getData()); + ColumnArray::Offsets & result_offsets = result_column->getOffsets(); + + ColumnArray::Offset current_offset = 0; + + for (size_t i = 0; i < input_rows_count; ++i) + { + StringRefs res = implCached(data[i]); + for (auto & r : res) + result_strings.insertData(r.data, r.size); + current_offset += res.size(); + result_offsets.push_back(current_offset); + } + + return result_column; + } + + void setResult(StringRefs & result, const Dwarf::LocationInfo & location, const std::vector & inline_frames) const override + { + + appendLocationToResult(result, location, nullptr); + for (const auto & inline_frame : inline_frames) + appendLocationToResult(result, inline_frame.location, &inline_frame); + } +private: + + inline ALWAYS_INLINE void appendLocationToResult(StringRefs & result, const Dwarf::LocationInfo & location, const Dwarf::SymbolizedFrame * frame) const + { + const char * arena_begin = nullptr; + WriteBufferFromArena out(cache.arena, arena_begin); + + writeString(location.file.toString(), out); + writeChar(':', out); + writeIntText(location.line, out); + + if (frame) + { + writeChar(':', out); + int status = 0; + writeString(demangle(frame->name, status), out); + } + + result.emplace_back(out.complete()); + } + +}; + +} + +void registerFunctionAddressToLineWithInlines(FunctionFactory & factory) +{ + factory.registerFunction(); +} + +} + +#endif diff --git a/src/Functions/array/arrayAUC.cpp b/src/Functions/array/arrayAUC.cpp index 510f52b0bed..9bebcf7fd8c 100644 --- a/src/Functions/array/arrayAUC.cpp +++ b/src/Functions/array/arrayAUC.cpp @@ -112,7 +112,8 @@ public: sorted_labels[i].label = label; } - std::sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; }); + /// Stable sort is required for for labels to apply in same order if score is equal + std::stable_sort(sorted_labels.begin(), sorted_labels.end(), [](const auto & lhs, const auto & rhs) { return lhs.score > rhs.score; }); /// We will first calculate non-normalized area. diff --git a/src/Functions/array/arrayAggregation.cpp b/src/Functions/array/arrayAggregation.cpp index da2304e1bb6..ee08c4f7f37 100644 --- a/src/Functions/array/arrayAggregation.cpp +++ b/src/Functions/array/arrayAggregation.cpp @@ -157,11 +157,11 @@ struct ArrayAggregateImpl return false; const AggregationType x = column_const->template getValue(); // NOLINT - const auto & data = checkAndGetColumn(&column_const->getDataColumn())->getData(); + const ColVecType * column_typed = checkAndGetColumn(&column_const->getDataColumn()); typename ColVecResultType::MutablePtr res_column; if constexpr (is_decimal) - res_column = ColVecResultType::create(offsets.size(), data.getScale()); + res_column = ColVecResultType::create(offsets.size(), column_typed->getScale()); else res_column = ColVecResultType::create(offsets.size()); @@ -185,7 +185,7 @@ struct ArrayAggregateImpl { if constexpr (is_decimal) { - res[i] = DecimalUtils::convertTo(x, data.getScale()); + res[i] = DecimalUtils::convertTo(x, column_typed->getScale()); } else { @@ -210,11 +210,11 @@ struct ArrayAggregateImpl throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Decimal math overflow"); } - auto result_scale = data.getScale() * array_size; + auto result_scale = column_typed->getScale() * array_size; if (unlikely(result_scale > DecimalUtils::max_precision)) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds", result_scale); - res[i] = DecimalUtils::convertTo(product, data.getScale() * array_size); + res[i] = DecimalUtils::convertTo(product, result_scale); } else { @@ -236,7 +236,7 @@ struct ArrayAggregateImpl typename ColVecResultType::MutablePtr res_column; if constexpr (is_decimal) - res_column = ColVecResultType::create(offsets.size(), data.getScale()); + res_column = ColVecResultType::create(offsets.size(), column->getScale()); else res_column = ColVecResultType::create(offsets.size()); @@ -309,7 +309,7 @@ struct ArrayAggregateImpl if constexpr (is_decimal) { aggregate_value = aggregate_value / AggregationType(count); - res[i] = DecimalUtils::convertTo(aggregate_value, data.getScale()); + res[i] = DecimalUtils::convertTo(aggregate_value, column->getScale()); } else { @@ -318,7 +318,7 @@ struct ArrayAggregateImpl } else if constexpr (aggregate_operation == AggregateOperation::product && is_decimal) { - auto result_scale = data.getScale() * count; + auto result_scale = column->getScale() * count; if (unlikely(result_scale > DecimalUtils::max_precision)) throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds", result_scale); diff --git a/src/Functions/array/arrayCompact.cpp b/src/Functions/array/arrayCompact.cpp index 7914b9a154e..c2908e37e12 100644 --- a/src/Functions/array/arrayCompact.cpp +++ b/src/Functions/array/arrayCompact.cpp @@ -40,7 +40,7 @@ struct ArrayCompactImpl typename ColVecType::MutablePtr res_values_column; if constexpr (is_decimal) - res_values_column = ColVecType::create(src_values.size(), src_values.getScale()); + res_values_column = ColVecType::create(src_values.size(), src_values_column->getScale()); else res_values_column = ColVecType::create(src_values.size()); diff --git a/src/Functions/array/arrayCumSum.cpp b/src/Functions/array/arrayCumSum.cpp index da8ef3d7852..467d9ad3951 100644 --- a/src/Functions/array/arrayCumSum.cpp +++ b/src/Functions/array/arrayCumSum.cpp @@ -101,9 +101,8 @@ struct ArrayCumSumImpl typename ColVecResult::MutablePtr res_nested; if constexpr (is_decimal) { - const typename ColVecType::Container & data = - checkAndGetColumn(&column_const->getDataColumn())->getData(); - res_nested = ColVecResult::create(0, data.getScale()); + const ColVecType * column_typed = checkAndGetColumn(&column_const->getDataColumn()); + res_nested = ColVecResult::create(0, column_typed->getScale()); } else res_nested = ColVecResult::create(); @@ -120,7 +119,7 @@ struct ArrayCumSumImpl typename ColVecResult::MutablePtr res_nested; if constexpr (is_decimal) - res_nested = ColVecResult::create(0, data.getScale()); + res_nested = ColVecResult::create(0, column->getScale()); else res_nested = ColVecResult::create(); diff --git a/src/Functions/array/arrayCumSumNonNegative.cpp b/src/Functions/array/arrayCumSumNonNegative.cpp index c40df27c1cc..476bbd08163 100644 --- a/src/Functions/array/arrayCumSumNonNegative.cpp +++ b/src/Functions/array/arrayCumSumNonNegative.cpp @@ -83,7 +83,7 @@ struct ArrayCumSumNonNegativeImpl typename ColVecResult::MutablePtr res_nested; if constexpr (is_decimal) - res_nested = ColVecResult::create(0, data.getScale()); + res_nested = ColVecResult::create(0, column->getScale()); else res_nested = ColVecResult::create(); diff --git a/src/Functions/array/arrayDifference.cpp b/src/Functions/array/arrayDifference.cpp index 97243f2cf74..c5fdf27100b 100644 --- a/src/Functions/array/arrayDifference.cpp +++ b/src/Functions/array/arrayDifference.cpp @@ -105,7 +105,7 @@ struct ArrayDifferenceImpl typename ColVecResult::MutablePtr res_nested; if constexpr (is_decimal) - res_nested = ColVecResult::create(0, data.getScale()); + res_nested = ColVecResult::create(0, column->getScale()); else res_nested = ColVecResult::create(); diff --git a/src/Functions/array/arraySort.cpp b/src/Functions/array/arraySort.cpp index 478c7e52614..476dfb46f07 100644 --- a/src/Functions/array/arraySort.cpp +++ b/src/Functions/array/arraySort.cpp @@ -1,4 +1,5 @@ #include "FunctionArrayMapped.h" +#include #include @@ -49,7 +50,7 @@ struct ArraySortImpl for (size_t i = 0; i < size; ++i) { auto next_offset = offsets[i]; - std::sort(&permutation[current_offset], &permutation[next_offset], Less(*mapped)); + ::sort(&permutation[current_offset], &permutation[next_offset], Less(*mapped)); current_offset = next_offset; } diff --git a/src/Functions/array/mapPopulateSeries.cpp b/src/Functions/array/mapPopulateSeries.cpp index b253a85c95d..17269f8dfe1 100644 --- a/src/Functions/array/mapPopulateSeries.cpp +++ b/src/Functions/array/mapPopulateSeries.cpp @@ -1,15 +1,20 @@ +#include + +#include #include #include #include #include +#include #include #include +#include +#include #include #include #include -#include "Core/ColumnWithTypeAndName.h" -#include "DataTypes/DataTypeMap.h" -#include "DataTypes/IDataType.h" +#include + namespace DB { @@ -19,6 +24,8 @@ namespace ErrorCodes extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int TOO_LARGE_ARRAY_SIZE; + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; } class FunctionMapPopulateSeries : public IFunction @@ -35,415 +42,458 @@ private: bool useDefaultImplementationForConstants() const override { return true; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; } - void checkTypes(const DataTypePtr & key_type, const DataTypePtr max_key_type) const + void checkTypes(const DataTypePtr & key_type, const DataTypePtr & value_type, const DataTypePtr & max_key_type) const { - WhichDataType which_key(key_type); - if (!(which_key.isInt() || which_key.isUInt())) + WhichDataType key_data_type(key_type); + WhichDataType value_data_type(value_type); + + if (!(key_data_type.isInt() || key_data_type.isUInt())) { throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Keys for {} function should be of integer type (signed or unsigned)", getName()); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Function {} key argument should be of signed or unsigned integer type. Actual type {}", + getName(), + key_type->getName()); } - if (max_key_type) + if (!(value_data_type.isInt() || value_data_type.isUInt())) { - WhichDataType which_max_key(max_key_type); - - if (which_max_key.isNullable()) - throw Exception( - ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Max key argument in arguments of function " + getName() + " can not be Nullable"); - - if (key_type->getTypeId() != max_key_type->getTypeId()) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Max key type in {} should be same as keys type", getName()); - } - } - - DataTypePtr getReturnTypeForTuple(const DataTypes & arguments) const - { - if (arguments.size() < 2) throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {} accepts at least two arrays for key and value", getName()); + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Function {} key argument should be of signed or unsigned integer type. Actual type {}", + getName(), + key_type->getName()); + } - if (arguments.size() > 3) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Too many arguments in {} call", getName()); + if (!max_key_type) + return; - const DataTypeArray * key_array_type = checkAndGetDataType(arguments[0].get()); - const DataTypeArray * val_array_type = checkAndGetDataType(arguments[1].get()); + WhichDataType max_key_data_type(max_key_type); - if (!key_array_type || !val_array_type) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} accepts two arrays for key and value", getName()); + if (max_key_data_type.isNullable()) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Function {} max key argument can not be Nullable. Actual type {}", + getName(), + max_key_type->getName()); - const auto & key_type = key_array_type->getNestedType(); - - if (arguments.size() == 3) - this->checkTypes(key_type, arguments[2]); - else - this->checkTypes(key_type, nullptr); - - return std::make_shared(DataTypes{arguments[0], arguments[1]}); - } - - DataTypePtr getReturnTypeForMap(const DataTypes & arguments) const - { - const auto * map = assert_cast(arguments[0].get()); - if (arguments.size() == 1) - this->checkTypes(map->getKeyType(), nullptr); - else if (arguments.size() == 2) - this->checkTypes(map->getKeyType(), arguments[1]); - else - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Too many arguments in {} call", getName()); - - return std::make_shared(map->getKeyType(), map->getValueType()); + if (!(max_key_data_type.isInt() || max_key_data_type.isUInt())) + throw Exception( + ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, + "Function {} max key should be of signed or unsigned integer type. Actual type {}.", + getName(), + key_type->getName(), + max_key_type->getName()); } DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override { - if (arguments.empty()) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, getName() + " accepts at least one map or two arrays"); + if (arguments.empty() || arguments.size() > 3) + throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Function {} accepts at least one map or two arrays arguments, and optional max key argument", + getName()); - if (arguments[0]->getTypeId() == TypeIndex::Array) - return getReturnTypeForTuple(arguments); - else if (arguments[0]->getTypeId() == TypeIndex::Map) - return getReturnTypeForMap(arguments); + WhichDataType key_argument_data_type(arguments[0]); + + DataTypePtr key_argument_series_type; + DataTypePtr value_argument_series_type; + + size_t max_key_argument_index = 0; + + if (key_argument_data_type.isArray()) + { + DataTypePtr value_type; + if (1 < arguments.size()) + value_type = arguments[1]; + + if (arguments.size() < 2 || (value_type && !isArray(value_type))) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Function {} if array argument is passed as key, additional array argument as value must be passed", + getName()); + + const auto & key_array_type = assert_cast(*arguments[0]); + const auto & value_array_type = assert_cast(*value_type); + + key_argument_series_type = key_array_type.getNestedType(); + value_argument_series_type = value_array_type.getNestedType(); + + max_key_argument_index = 2; + } + else if (key_argument_data_type.isMap()) + { + const auto & map_data_type = assert_cast(*arguments[0]); + + key_argument_series_type = map_data_type.getKeyType(); + value_argument_series_type = map_data_type.getValueType(); + + max_key_argument_index = 1; + } else throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Function {} only accepts one map or arrays, but got {}", getName(), arguments[0]->getName()); + + DataTypePtr max_key_argument_type; + if (max_key_argument_index < arguments.size()) + max_key_argument_type = arguments[max_key_argument_index]; + + checkTypes(key_argument_series_type, value_argument_series_type, max_key_argument_type); + + if (key_argument_data_type.isArray()) + return std::make_shared(DataTypes{arguments[0], arguments[1]}); + else + return arguments[0]; } - // Struct holds input and output columns references, - // Both arrays and maps have similar columns to work with but extracted differently - template - struct ColumnsInOut + template + void executeImplTyped( + const ColumnPtr & key_column, + const ColumnPtr & value_column, + const ColumnPtr & offsets_column, + const ColumnPtr & max_key_column, + MutableColumnPtr result_key_column, + MutableColumnPtr result_value_column, + MutableColumnPtr result_offset_column) const { - // inputs - const PaddedPODArray & in_keys_data; - const PaddedPODArray & in_vals_data; - const IColumn::Offsets & in_key_offsets; - const IColumn::Offsets & in_val_offsets; - size_t row_count; - bool key_is_const; - bool val_is_const; + const auto & key_column_typed = assert_cast &>(*key_column); + const auto & key_column_data = key_column_typed.getData(); - // outputs - PaddedPODArray & out_keys_data; - PaddedPODArray & out_vals_data; + const auto & offsets_column_typed = assert_cast &>(*offsets_column); + const auto & offsets = offsets_column_typed.getData(); - IColumn::Offsets & out_keys_offsets; - // with map argument this field will not be used - IColumn::Offsets * out_vals_offsets; - }; + const auto & value_column_typed = assert_cast &>(*value_column); + const auto & value_column_data = value_column_typed.getData(); - template - ColumnsInOut getInOutDataFromArrays(MutableColumnPtr & res_column, ColumnPtr * arg_columns) const - { - auto * out_tuple = assert_cast(res_column.get()); - auto & out_keys_array = assert_cast(out_tuple->getColumn(0)); - auto & out_vals_array = assert_cast(out_tuple->getColumn(1)); + auto & result_key_column_typed = assert_cast &>(*result_key_column); + auto & result_key_data = result_key_column_typed.getData(); - const auto * key_column = arg_columns[0].get(); - const auto * in_keys_array = checkAndGetColumn(key_column); + auto & result_value_column_typed = assert_cast &>(*result_value_column); + auto & result_value_data = result_value_column_typed.getData(); - bool key_is_const = false, val_is_const = false; + auto & result_offsets_column_typed = assert_cast &>(*result_offset_column); + auto & result_offsets_data = result_offsets_column_typed.getData(); - if (!in_keys_array) + const PaddedPODArray * max_key_data = max_key_column ? &assert_cast &>(*max_key_column).getData() : nullptr; + + PaddedPODArray> sorted_keys_values; + + size_t key_offsets_size = offsets.size(); + result_key_data.reserve(key_offsets_size); + result_value_data.reserve(key_offsets_size); + + for (size_t offset_index = 0; offset_index < key_offsets_size; ++offset_index) { - const ColumnConst * const_array = checkAndGetColumnConst(key_column); - if (!const_array) - throw Exception( - ErrorCodes::ILLEGAL_COLUMN, "Expected array column in function {}, found {}", getName(), key_column->getName()); + size_t start_offset = offsets[offset_index - 1]; + size_t end_offset = offsets[offset_index]; - in_keys_array = checkAndGetColumn(const_array->getDataColumnPtr().get()); - key_is_const = true; - } + sorted_keys_values.clear(); - const auto * val_column = arg_columns[1].get(); - const auto * in_values_array = checkAndGetColumn(val_column); - if (!in_values_array) - { - const ColumnConst * const_array = checkAndGetColumnConst(val_column); - if (!const_array) - throw Exception( - ErrorCodes::ILLEGAL_COLUMN, "Expected array column in function {}, found {}", getName(), val_column->getName()); + for (; start_offset < end_offset; ++start_offset) + sorted_keys_values.emplace_back(key_column_data[start_offset], value_column_data[start_offset]); - in_values_array = checkAndGetColumn(const_array->getDataColumnPtr().get()); - val_is_const = true; - } - - if (!in_keys_array || !in_values_array) - /* something went wrong */ - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal columns in arguments of function " + getName()); - - const auto & in_keys_data = assert_cast &>(in_keys_array->getData()).getData(); - const auto & in_values_data = assert_cast &>(in_values_array->getData()).getData(); - const auto & in_keys_offsets = in_keys_array->getOffsets(); - const auto & in_vals_offsets = in_values_array->getOffsets(); - - auto & out_keys_data = assert_cast &>(out_keys_array.getData()).getData(); - auto & out_vals_data = assert_cast &>(out_vals_array.getData()).getData(); - auto & out_keys_offsets = out_keys_array.getOffsets(); - - size_t row_count = key_is_const ? in_values_array->size() : in_keys_array->size(); - IColumn::Offsets * out_vals_offsets = &out_vals_array.getOffsets(); - - return { - in_keys_data, - in_values_data, - in_keys_offsets, - in_vals_offsets, - row_count, - key_is_const, - val_is_const, - out_keys_data, - out_vals_data, - out_keys_offsets, - out_vals_offsets}; - } - - template - ColumnsInOut getInOutDataFromMap(MutableColumnPtr & res_column, ColumnPtr * arg_columns) const - { - const auto * in_map = assert_cast(arg_columns[0].get()); - const auto & in_nested_array = in_map->getNestedColumn(); - const auto & in_nested_tuple = in_map->getNestedData(); - const auto & in_keys_data = assert_cast &>(in_nested_tuple.getColumn(0)).getData(); - const auto & in_vals_data = assert_cast &>(in_nested_tuple.getColumn(1)).getData(); - const auto & in_keys_offsets = in_nested_array.getOffsets(); - - auto * out_map = assert_cast(res_column.get()); - auto & out_nested_array = out_map->getNestedColumn(); - auto & out_nested_tuple = out_map->getNestedData(); - auto & out_keys_data = assert_cast &>(out_nested_tuple.getColumn(0)).getData(); - auto & out_vals_data = assert_cast &>(out_nested_tuple.getColumn(1)).getData(); - auto & out_keys_offsets = out_nested_array.getOffsets(); - - return { - in_keys_data, - in_vals_data, - in_keys_offsets, - in_keys_offsets, - in_nested_array.size(), - false, - false, - out_keys_data, - out_vals_data, - out_keys_offsets, - nullptr}; - } - - template - ColumnPtr execute2(ColumnPtr * arg_columns, ColumnPtr max_key_column, const DataTypePtr & res_type) const - { - MutableColumnPtr res_column = res_type->createColumn(); - bool max_key_is_const = false; - auto columns = res_column->getDataType() == TypeIndex::Tuple ? getInOutDataFromArrays(res_column, arg_columns) - : getInOutDataFromMap(res_column, arg_columns); - - KeyType max_key_const{0}; - - if (max_key_column && isColumnConst(*max_key_column)) - { - const auto * column_const = static_cast(&*max_key_column); - max_key_const = column_const->template getValue(); - max_key_is_const = true; - } - - IColumn::Offset offset{0}; - std::map res_map; - - //Iterate through two arrays and fill result values. - for (size_t row = 0; row < columns.row_count; ++row) - { - size_t key_offset = 0, val_offset = 0, items_count = columns.in_key_offsets[0], val_array_size = columns.in_val_offsets[0]; - - res_map.clear(); - - if (!columns.key_is_const) + if unlikely(sorted_keys_values.empty()) { - key_offset = row > 0 ? columns.in_key_offsets[row - 1] : 0; - items_count = columns.in_key_offsets[row] - key_offset; - } - - if (!columns.val_is_const) - { - val_offset = row > 0 ? columns.in_val_offsets[row - 1] : 0; - val_array_size = columns.in_val_offsets[row] - val_offset; - } - - if (items_count != val_array_size) - throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Key and value array should have same amount of elements in function {}", - getName()); - - if (items_count == 0) - { - columns.out_keys_offsets.push_back(offset); + result_offsets_data.emplace_back(result_value_data.size()); continue; } - for (size_t i = 0; i < items_count; ++i) + ::sort(sorted_keys_values.begin(), sorted_keys_values.end()); + + KeyType min_key = sorted_keys_values.front().first; + KeyType max_key = sorted_keys_values.back().first; + + if (max_key_data) { - res_map.insert({columns.in_keys_data[key_offset + i], columns.in_vals_data[val_offset + i]}); - } + max_key = (*max_key_data)[offset_index]; - auto min_key = res_map.begin()->first; - auto max_key = res_map.rbegin()->first; - - if (max_key_column) - { - /* update the current max key if it's not constant */ - if (max_key_is_const) + if (unlikely(max_key < min_key)) { - max_key = max_key_const; - } - else - { - max_key = (static_cast *>(max_key_column.get()))->getData()[row]; - } - - /* no need to add anything, max key is less that first key */ - if (max_key < min_key) - { - columns.out_keys_offsets.push_back(offset); + result_offsets_data.emplace_back(result_value_data.size()); continue; } } - static constexpr size_t MAX_ARRAY_SIZE = 1ULL << 30; - if (static_cast(max_key) - static_cast(min_key) > MAX_ARRAY_SIZE) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Too large array size in the result of function {}", getName()); + using KeyTypeUnsigned = ::make_unsigned_t; + KeyTypeUnsigned max_min_key_difference = 0; - /* fill the result arrays */ - KeyType key; - for (key = min_key;; ++key) + if constexpr (::is_unsigned_v) { - columns.out_keys_data.push_back(key); + max_min_key_difference = max_key - min_key; + } + else + { + bool is_max_key_positive = max_key >= 0; + bool is_min_key_positive = min_key >= 0; - auto it = res_map.find(key); - if (it != res_map.end()) + if (is_max_key_positive && is_min_key_positive) { - columns.out_vals_data.push_back(it->second); + max_min_key_difference = static_cast(max_key - min_key); + } + else if (is_max_key_positive && !is_min_key_positive) + { + KeyTypeUnsigned min_key_unsigned = -static_cast(min_key); + max_min_key_difference = static_cast(max_key) + min_key_unsigned; } else { - columns.out_vals_data.push_back(0); + /// Both max and min key are negative + KeyTypeUnsigned min_key_unsigned = -static_cast(min_key); + KeyTypeUnsigned max_key_unsigned = -static_cast(max_key); + max_min_key_difference = min_key_unsigned - max_key_unsigned; + } + } + + static constexpr size_t MAX_ARRAY_SIZE = 1ULL << 30; + if (max_min_key_difference > MAX_ARRAY_SIZE) + throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Function {} too large array size in the result", + getName()); + + size_t length = static_cast(max_min_key_difference); + size_t result_key_data_size = result_key_data.size(); + size_t result_value_data_size = result_value_data.size(); + size_t sorted_keys_values_size = sorted_keys_values.size(); + + result_key_data.resize_fill(result_key_data_size + length + 1); + result_value_data.resize_fill(result_value_data_size + length + 1); + + size_t sorted_values_index = 0; + + for (KeyType current_key = min_key; current_key <= max_key; ++current_key) + { + size_t key_offset_index = current_key - min_key; + size_t insert_index = result_value_data_size + key_offset_index; + + result_key_data[insert_index] = current_key; + + if (sorted_values_index < sorted_keys_values_size && + sorted_keys_values[sorted_values_index].first == current_key) + { + auto & sorted_key_value = sorted_keys_values[sorted_values_index]; + if (current_key == sorted_key_value.first) + { + result_value_data[insert_index] = sorted_key_value.second; + } + + ++sorted_values_index; + while (sorted_values_index < sorted_keys_values_size && + current_key == sorted_keys_values[sorted_values_index].first) + { + ++sorted_values_index; + } } - ++offset; - if (key == max_key) + if (current_key == max_key) break; } - columns.out_keys_offsets.push_back(offset); - } - - if (columns.out_vals_offsets) - columns.out_vals_offsets->insert(columns.out_keys_offsets.begin(), columns.out_keys_offsets.end()); - - return res_column; - } - - template - ColumnPtr execute1(ColumnPtr * arg_columns, ColumnPtr max_key_column, const DataTypePtr & res_type, const DataTypePtr & val_type) const - { - switch (val_type->getTypeId()) - { - case TypeIndex::Int8: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::Int16: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::Int32: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::Int64: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::Int128: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::Int256: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::UInt8: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::UInt16: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::UInt32: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::UInt64: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::UInt128: - return execute2(arg_columns, max_key_column, res_type); - case TypeIndex::UInt256: - return execute2(arg_columns, max_key_column, res_type); - default: - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal columns in arguments of function " + getName()); + result_offsets_data.emplace_back(result_value_data.size()); } } - ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t) const override + struct KeyAndValueInput { - DataTypePtr res_type, key_type, val_type; - ColumnPtr max_key_column = nullptr; - ColumnPtr arg_columns[] = {arguments[0].column, nullptr}; + DataTypePtr key_series_type; + DataTypePtr value_series_type; - if (arguments[0].type->getTypeId() == TypeIndex::Array) + ColumnPtr key_column; + ColumnPtr value_column; + ColumnPtr offsets_column; + + /// Optional max key column + ColumnPtr max_key_column; + }; + + KeyAndValueInput extractKeyAndValueInput(const ColumnsWithTypeAndName & arguments) const + { + KeyAndValueInput input; + + size_t max_key_argument_index = 0; + + auto first_argument_column = arguments[0].column->convertToFullColumnIfConst(); + ColumnPtr second_argument_array_column; + + if (const auto * key_argument_array_column = typeid_cast(first_argument_column.get())) { - key_type = assert_cast(arguments[0].type.get())->getNestedType(); - val_type = assert_cast(arguments[1].type.get())->getNestedType(); - res_type = getReturnTypeImpl(DataTypes{arguments[0].type, arguments[1].type}); + const ColumnArray * value_argument_array_column = nullptr; - arg_columns[1] = arguments[1].column; - if (arguments.size() == 3) + if (1 < arguments.size()) { - /* max key provided */ - max_key_column = arguments[2].column; + second_argument_array_column = arguments[1].column->convertToFullColumnIfConst(); + value_argument_array_column = typeid_cast(second_argument_array_column.get()); } + + if (!value_argument_array_column) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Function {} if array argument is passed as key, additional array argument as value must be passed", + getName()); + + input.key_series_type = assert_cast(*arguments[0].type).getNestedType(); + input.key_column = key_argument_array_column->getDataPtr(); + const auto & key_offsets = key_argument_array_column->getOffsets(); + + input.value_series_type = assert_cast(*arguments[1].type).getNestedType(); + input.value_column = value_argument_array_column->getDataPtr(); + const auto & value_offsets = value_argument_array_column->getOffsets(); + + if (key_offsets != value_offsets) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Function {} key and value array should have same amount of elements", + getName()); + + input.offsets_column = key_argument_array_column->getOffsetsPtr(); + max_key_argument_index = 2; + } + else if (const auto * key_argument_map_column = typeid_cast(first_argument_column.get())) + { + const auto & nested_array = key_argument_map_column->getNestedColumn(); + const auto & nested_data_column = key_argument_map_column->getNestedData(); + + const auto & map_argument_type = assert_cast(*arguments[0].type); + input.key_series_type = map_argument_type.getKeyType(); + input.value_series_type = map_argument_type.getValueType(); + + input.key_column = nested_data_column.getColumnPtr(0); + input.value_column = nested_data_column.getColumnPtr(1); + input.offsets_column = nested_array.getOffsetsPtr(); + + max_key_argument_index = 1; + } + else + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Function {} only accepts one map or arrays, but got {}", + getName(), + arguments[0].type->getName()); + + ColumnPtr max_key_column; + + if (max_key_argument_index < arguments.size()) + { + max_key_column = arguments[max_key_argument_index].column->convertToFullColumnIfConst(); + auto max_key_column_type = arguments[max_key_argument_index].type; + + if (!max_key_column_type->equals(*input.key_series_type)) + { + ColumnWithTypeAndName column_to_cast = {max_key_column, max_key_column_type, ""}; + auto casted_column = castColumnAccurate(std::move(column_to_cast), input.key_series_type); + max_key_column = std::move(casted_column); + } + } + + input.max_key_column = std::move(max_key_column); + + return input; + } + + struct ResultColumns + { + MutableColumnPtr result_key_column; + MutableColumnPtr result_value_column; + MutableColumnPtr result_offset_column; + IColumn * result_offset_column_raw; + /// If we return tuple of two arrays, this offset need to be the same as result_offset_column + MutableColumnPtr result_array_additional_offset_column; + }; + + ResultColumns extractResultColumns(MutableColumnPtr & result_column, const DataTypePtr & result_type) const + { + ResultColumns result; + + auto * tuple_column = typeid_cast(result_column.get()); + + if (tuple_column && tuple_column->tupleSize() == 2) + { + auto key_array_column = tuple_column->getColumnPtr(0)->assumeMutable(); + auto value_array_column = tuple_column->getColumnPtr(1)->assumeMutable(); + + auto * key_array_column_typed = typeid_cast(key_array_column.get()); + auto * value_array_column_typed = typeid_cast(value_array_column.get()); + + if (!key_array_column_typed || !value_array_column_typed) + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Function {} result type should be Tuple with two nested Array columns or Map. Actual {}", + getName(), + result_type->getName()); + + result.result_key_column = key_array_column_typed->getDataPtr()->assumeMutable(); + result.result_value_column = value_array_column_typed->getDataPtr()->assumeMutable(); + result.result_offset_column = key_array_column_typed->getOffsetsPtr()->assumeMutable(); + result.result_offset_column_raw = result.result_offset_column.get(); + result.result_array_additional_offset_column = value_array_column_typed->getOffsetsPtr()->assumeMutable(); + } + else if (const auto * map_column = typeid_cast(result_column.get())) + { + result.result_key_column = map_column->getNestedData().getColumnPtr(0)->assumeMutable(); + result.result_value_column = map_column->getNestedData().getColumnPtr(1)->assumeMutable(); + result.result_offset_column = map_column->getNestedColumn().getOffsetsPtr()->assumeMutable(); + result.result_offset_column_raw = result.result_offset_column.get(); + result.result_array_additional_offset_column = nullptr; } else { - assert(arguments[0].type->getTypeId() == TypeIndex::Map); - - const auto * map_type = assert_cast(arguments[0].type.get()); - res_type = getReturnTypeImpl(DataTypes{arguments[0].type}); - key_type = map_type->getKeyType(); - val_type = map_type->getValueType(); - - if (arguments.size() == 2) - { - /* max key provided */ - max_key_column = arguments[1].column; - } + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Function {} result type should be Tuple with two nested Array columns or Map. Actual {}", + getName(), + result_type->getName()); } - switch (key_type->getTypeId()) + return result; + } + + ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t) const override + { + auto input = extractKeyAndValueInput(arguments); + + auto result_column = result_type->createColumn(); + auto result_columns = extractResultColumns(result_column, result_type); + + auto call = [&](const auto & types) { - case TypeIndex::Int8: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::Int16: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::Int32: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::Int64: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::Int128: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::Int256: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::UInt8: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::UInt16: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::UInt32: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::UInt64: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::UInt128: - return execute1(arg_columns, max_key_column, res_type, val_type); - case TypeIndex::UInt256: - return execute1(arg_columns, max_key_column, res_type, val_type); - default: - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Illegal columns in arguments of function " + getName()); + using Types = std::decay_t; + using KeyType = typename Types::LeftType; + using ValueType = typename Types::RightType; + + static constexpr bool key_and_value_are_numbers = IsDataTypeNumber && IsDataTypeNumber; + static constexpr bool key_is_float = std::is_same_v || std::is_same_v; + + if constexpr (key_and_value_are_numbers && !key_is_float) + { + using KeyFieldType = typename KeyType::FieldType; + using ValueFieldType = typename ValueType::FieldType; + + executeImplTyped( + input.key_column, + input.value_column, + input.offsets_column, + input.max_key_column, + std::move(result_columns.result_key_column), + std::move(result_columns.result_value_column), + std::move(result_columns.result_offset_column)); + + return true; + } + + return false; + }; + + if (!callOnTwoTypeIndexes(input.key_series_type->getTypeId(), input.value_series_type->getTypeId(), call)) + throw Exception(ErrorCodes::ILLEGAL_COLUMN, + "Function {} illegal columns passed as arguments", + getName()); + + if (result_columns.result_array_additional_offset_column) + { + result_columns.result_array_additional_offset_column->insertRangeFrom( + *result_columns.result_offset_column_raw, + 0, + result_columns.result_offset_column_raw->size()); } + + return result_column; } }; @@ -451,4 +501,5 @@ void registerFunctionMapPopulateSeries(FunctionFactory & factory) { factory.registerFunction(); } + } diff --git a/src/Functions/currentRoles.cpp b/src/Functions/currentRoles.cpp index f176f51f6c9..46ddb77c321 100644 --- a/src/Functions/currentRoles.cpp +++ b/src/Functions/currentRoles.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -53,7 +54,7 @@ namespace } /// We sort the names because the result of the function should not depend on the order of UUIDs. - std::sort(role_names.begin(), role_names.end()); + ::sort(role_names.begin(), role_names.end()); } size_t getNumberOfArguments() const override { return 0; } diff --git a/src/Functions/dateName.cpp b/src/Functions/dateName.cpp index c89a7f80dfd..eef9bc3955b 100644 --- a/src/Functions/dateName.cpp +++ b/src/Functions/dateName.cpp @@ -148,7 +148,7 @@ public: UInt32 scale [[maybe_unused]] = 0; if constexpr (std::is_same_v) { - scale = times_data.getScale(); + scale = times->getScale(); } auto result_column = ColumnString::create(); diff --git a/src/Functions/formatDateTime.cpp b/src/Functions/formatDateTime.cpp index 9f303b86ad3..e2ec90f4e61 100644 --- a/src/Functions/formatDateTime.cpp +++ b/src/Functions/formatDateTime.cpp @@ -440,7 +440,7 @@ public: UInt32 scale [[maybe_unused]] = 0; if constexpr (std::is_same_v) { - scale = vec.getScale(); + scale = times->getScale(); } auto col_res = ColumnString::create(); diff --git a/src/Functions/geoToH3.cpp b/src/Functions/geoToH3.cpp index 18951d1a03f..fb7301de776 100644 --- a/src/Functions/geoToH3.cpp +++ b/src/Functions/geoToH3.cpp @@ -11,6 +11,7 @@ #include #include +#include #include @@ -20,6 +21,8 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int INCORRECT_DATA; + extern const int ILLEGAL_COLUMN; + extern const int ARGUMENT_OUT_OF_BOUND; } namespace @@ -68,9 +71,35 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lon = arguments[0].column.get(); - const auto * col_lat = arguments[1].column.get(); - const auto * col_res = arguments[2].column.get(); + const auto * col_lon = checkAndGetColumn(arguments[0].column.get()); + if (!col_lon) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64.", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_lon = col_lon->getData(); + + const auto * col_lat = checkAndGetColumn(arguments[1].column.get()); + if (!col_lat) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64.", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_lat = col_lat->getData(); + + const auto * col_res = checkAndGetColumn(arguments[2].column.get()); + if (!col_res) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt8.", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_res = col_res->getData(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); @@ -78,9 +107,17 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const double lon = col_lon->getFloat64(row); - const double lat = col_lat->getFloat64(row); - const UInt8 res = col_res->getUInt(row); + const double lon = data_lon[row]; + const double lat = data_lat[row]; + const UInt8 res = data_res[row]; + + if (res > MAX_H3_RES) + throw Exception( + ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "The argument 'resolution' ({}) of function {} is out of bounds because the maximum resolution in H3 library is ", + toString(res), + getName(), + MAX_H3_RES); LatLng coord; coord.lng = degsToRads(lon); diff --git a/src/Functions/geoToS2.cpp b/src/Functions/geoToS2.cpp index 32d2a1d7a10..d69c15bdbe0 100644 --- a/src/Functions/geoToS2.cpp +++ b/src/Functions/geoToS2.cpp @@ -19,6 +19,7 @@ namespace DB namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int ILLEGAL_COLUMN; } namespace @@ -66,8 +67,25 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lon = arguments[0].column.get(); - const auto * col_lat = arguments[1].column.get(); + const auto * col_lon = checkAndGetColumn(arguments[0].column.get()); + if (!col_lon) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_col_lon = col_lon->getData(); + + const auto * col_lat = checkAndGetColumn(arguments[1].column.get()); + if (!col_lat) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[0].type->getName(), + 2, + getName()); + const auto & data_col_lat = col_lat->getData(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); @@ -75,16 +93,14 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const Float64 lon = col_lon->getFloat64(row); - const Float64 lat = col_lat->getFloat64(row); + const Float64 lon = data_col_lon[row]; + const Float64 lat = data_col_lat[row]; if (isNaN(lon) || isNaN(lat)) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Arguments must not be NaN"); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments must not be NaN"); if (!(isFinite(lon) && isFinite(lat))) - throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Arguments must not be infinite"); + throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Arguments must not be infinite"); /// S2 acceptes point as (latitude, longitude) S2LatLng lat_lng = S2LatLng::FromDegrees(lat, lon); @@ -95,7 +111,6 @@ public: return dst; } - }; } @@ -105,7 +120,6 @@ void registerFunctionGeoToS2(FunctionFactory & factory) factory.registerFunction(); } - } #endif diff --git a/src/Functions/logTrace.cpp b/src/Functions/logTrace.cpp index acf2a2041ec..05315f4dff6 100644 --- a/src/Functions/logTrace.cpp +++ b/src/Functions/logTrace.cpp @@ -48,7 +48,7 @@ namespace "First argument for function " + getName() + " must be Constant string", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT); static auto * log = &Poco::Logger::get("FunctionLogTrace"); - LOG_TRACE(log, message); + LOG_TRACE(log, fmt::runtime(message)); return DataTypeUInt8().createColumnConst(input_rows_count, 0); } diff --git a/src/Functions/match.cpp b/src/Functions/match.cpp index 31d36577445..69dc1a3d99a 100644 --- a/src/Functions/match.cpp +++ b/src/Functions/match.cpp @@ -20,6 +20,7 @@ using FunctionMatch = FunctionsStringSearch>; void registerFunctionMatch(FunctionFactory & factory) { factory.registerFunction(); + factory.registerAlias("REGEXP_MATCHES", NameMatch::name, FunctionFactory::CaseInsensitive); } } diff --git a/src/Functions/registerFunctionsIntrospection.cpp b/src/Functions/registerFunctionsIntrospection.cpp index fe76c96d62d..76a92847d8e 100644 --- a/src/Functions/registerFunctionsIntrospection.cpp +++ b/src/Functions/registerFunctionsIntrospection.cpp @@ -6,6 +6,7 @@ class FunctionFactory; #if defined(OS_LINUX) void registerFunctionAddressToSymbol(FunctionFactory & factory); void registerFunctionAddressToLine(FunctionFactory & factory); +void registerFunctionAddressToLineWithInlines(FunctionFactory & factory); #endif void registerFunctionDemangle(FunctionFactory & factory); @@ -17,6 +18,7 @@ void registerFunctionsIntrospection(FunctionFactory & factory) #if defined(OS_LINUX) registerFunctionAddressToSymbol(factory); registerFunctionAddressToLine(factory); + registerFunctionAddressToLineWithInlines(factory); #endif registerFunctionDemangle(factory); registerFunctionTrap(factory); diff --git a/src/Functions/replaceAll.cpp b/src/Functions/replaceAll.cpp index cc29e57ea69..25a5b33c3a0 100644 --- a/src/Functions/replaceAll.cpp +++ b/src/Functions/replaceAll.cpp @@ -21,6 +21,7 @@ void registerFunctionReplaceAll(FunctionFactory & factory) { factory.registerFunction(); factory.registerAlias("replace", NameReplaceAll::name, FunctionFactory::CaseInsensitive); + factory.registerAlias("REGEXP_REPLACE", NameReplaceAll::name, FunctionFactory::CaseInsensitive); } } diff --git a/src/Functions/s2CapContains.cpp b/src/Functions/s2CapContains.cpp index 100b028646c..6604ff9707e 100644 --- a/src/Functions/s2CapContains.cpp +++ b/src/Functions/s2CapContains.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } namespace @@ -83,19 +84,47 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_center = arguments[0].column.get(); - const auto * col_degrees = arguments[1].column.get(); - const auto * col_point = arguments[2].column.get(); + const auto * col_center = checkAndGetColumn(arguments[0].column.get()); + if (!col_center) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_center = col_center->getData(); + + const auto * col_degrees = checkAndGetColumn(arguments[1].column.get()); + if (!col_degrees) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_degrees = col_degrees->getData(); + + + const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + if (!col_point) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_point = col_point->getData(); + auto dst = ColumnUInt8::create(); auto & dst_data = dst->getData(); dst_data.reserve(input_rows_count); - for (size_t row=0 ; row < input_rows_count; ++row) + for (size_t row = 0; row < input_rows_count; ++row) { - const auto center = S2CellId(col_center->getUInt(row)); - const Float64 degrees = col_degrees->getFloat64(row); - const auto point = S2CellId(col_point->getUInt(row)); + const auto center = S2CellId(data_center[row]); + const Float64 degrees = data_degrees[row]; + const auto point = S2CellId(data_point[row]); if (isNaN(degrees)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be nan"); diff --git a/src/Functions/s2CapUnion.cpp b/src/Functions/s2CapUnion.cpp index 263163963af..7af6324a7d5 100644 --- a/src/Functions/s2CapUnion.cpp +++ b/src/Functions/s2CapUnion.cpp @@ -20,6 +20,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } namespace @@ -81,10 +82,45 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_center1 = arguments[0].column.get(); - const auto * col_radius1 = arguments[1].column.get(); - const auto * col_center2 = arguments[2].column.get(); - const auto * col_radius2 = arguments[3].column.get(); + const auto * col_center1 = checkAndGetColumn(arguments[0].column.get()); + if (!col_center1) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_center1 = col_center1->getData(); + + const auto * col_radius1 = checkAndGetColumn(arguments[1].column.get()); + if (!col_radius1) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_radius1 = col_radius1->getData(); + + const auto * col_center2 = checkAndGetColumn(arguments[2].column.get()); + if (!col_center2) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_center2 = col_center2->getData(); + + const auto * col_radius2 = checkAndGetColumn(arguments[3].column.get()); + if (!col_radius2) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be Float64", + arguments[3].type->getName(), + 4, + getName()); + const auto & data_radius2 = col_radius2->getData(); auto col_res_center = ColumnUInt64::create(); auto col_res_radius = ColumnFloat64::create(); @@ -97,10 +133,10 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const UInt64 first_center = col_center1->getUInt(row); - const Float64 first_radius = col_radius1->getFloat64(row); - const UInt64 second_center = col_center2->getUInt(row); - const Float64 second_radius = col_radius2->getFloat64(row); + const UInt64 first_center = data_center1[row]; + const Float64 first_radius = data_radius1[row]; + const UInt64 second_center = data_center2[row]; + const Float64 second_radius = data_radius2[row]; if (isNaN(first_radius) || isNaN(second_radius)) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Radius of the cap must not be nan"); @@ -125,7 +161,6 @@ public: return ColumnTuple::create(Columns{std::move(col_res_center), std::move(col_res_radius)}); } - }; } @@ -135,7 +170,6 @@ void registerFunctionS2CapUnion(FunctionFactory & factory) factory.registerFunction(); } - } #endif diff --git a/src/Functions/s2CellsIntersect.cpp b/src/Functions/s2CellsIntersect.cpp index f8273a1fcca..5d9796ea26c 100644 --- a/src/Functions/s2CellsIntersect.cpp +++ b/src/Functions/s2CellsIntersect.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } namespace @@ -65,8 +66,25 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id_first = arguments[0].column.get(); - const auto * col_id_second = arguments[1].column.get(); + const auto * col_id_first = checkAndGetColumn(arguments[0].column.get()); + if (!col_id_first) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_id_first = col_id_first->getData(); + + const auto * col_id_second = checkAndGetColumn(arguments[1].column.get()); + if (!col_id_second) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_id_second = col_id_second->getData(); auto dst = ColumnUInt8::create(); auto & dst_data = dst->getData(); @@ -74,8 +92,8 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const UInt64 id_first = col_id_first->getInt(row); - const UInt64 id_second = col_id_second->getInt(row); + const UInt64 id_first = data_id_first[row]; + const UInt64 id_second = data_id_second[row]; auto first_cell = S2CellId(id_first); auto second_cell = S2CellId(id_second); diff --git a/src/Functions/s2GetNeighbors.cpp b/src/Functions/s2GetNeighbors.cpp index c0b2e634e6f..32eaff90740 100644 --- a/src/Functions/s2GetNeighbors.cpp +++ b/src/Functions/s2GetNeighbors.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } namespace @@ -56,15 +57,25 @@ public: if (!WhichDataType(arg).isUInt64()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of argument {} of function {}. Must be Float64", - arg->getName(), 1, getName()); + "Illegal type {} of argument {} of function {}. Must be UInt64", + arg->getName(), + 1, + getName()); return std::make_shared(std::make_shared()); } ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id = arguments[0].column.get(); + const auto * col_id = checkAndGetColumn(arguments[0].column.get()); + if (!col_id) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_id = col_id->getData(); auto dst = ColumnArray::create(ColumnUInt64::create()); auto & dst_data = dst->getData(); @@ -74,7 +85,7 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const UInt64 id = col_id->getUInt(row); + const UInt64 id = data_id[row]; S2CellId cell_id(id); diff --git a/src/Functions/s2RectAdd.cpp b/src/Functions/s2RectAdd.cpp index f7c39b2a6b1..75da7de8f7e 100644 --- a/src/Functions/s2RectAdd.cpp +++ b/src/Functions/s2RectAdd.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } namespace @@ -64,9 +65,35 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo = arguments[0].column.get(); - const auto * col_hi = arguments[1].column.get(); - const auto * col_point = arguments[2].column.get(); + const auto * col_lo = checkAndGetColumn(arguments[0].column.get()); + if (!col_lo) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_low = col_lo->getData(); + + const auto * col_hi = checkAndGetColumn(arguments[1].column.get()); + if (!col_hi) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_hi = col_hi->getData(); + + const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + if (!col_point) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_point = col_point->getData(); auto col_res_first = ColumnUInt64::create(); auto col_res_second = ColumnUInt64::create(); @@ -79,9 +106,9 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const auto lo = S2CellId(col_lo->getUInt(row)); - const auto hi = S2CellId(col_hi->getUInt(row)); - const auto point = S2CellId(col_point->getUInt(row)); + const auto lo = S2CellId(data_low[row]); + const auto hi = S2CellId(data_hi[row]); + const auto point = S2CellId(data_point[row]); if (!lo.is_valid() || !hi.is_valid()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Rectangle is not valid"); diff --git a/src/Functions/s2RectContains.cpp b/src/Functions/s2RectContains.cpp index 90ced5450bc..be46253f70e 100644 --- a/src/Functions/s2RectContains.cpp +++ b/src/Functions/s2RectContains.cpp @@ -14,11 +14,12 @@ namespace DB { - namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; + } namespace @@ -62,9 +63,35 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo = arguments[0].column.get(); - const auto * col_hi = arguments[1].column.get(); - const auto * col_point = arguments[2].column.get(); + const auto * col_lo = checkAndGetColumn(arguments[0].column.get()); + if (!col_lo) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_low = col_lo->getData(); + + const auto * col_hi = checkAndGetColumn(arguments[1].column.get()); + if (!col_hi) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_hi = col_hi->getData(); + + const auto * col_point = checkAndGetColumn(arguments[2].column.get()); + if (!col_point) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_point = col_point->getData(); auto dst = ColumnVector::create(); auto & dst_data = dst->getData(); @@ -72,9 +99,9 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const auto lo = S2CellId(col_lo->getUInt(row)); - const auto hi = S2CellId(col_hi->getUInt(row)); - const auto point = S2CellId(col_point->getUInt(row)); + const auto lo = S2CellId(data_low[row]); + const auto hi = S2CellId(data_hi[row]); + const auto point = S2CellId(data_point[row]); if (!lo.is_valid() || !hi.is_valid()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Rectangle is not valid"); diff --git a/src/Functions/s2RectIntersection.cpp b/src/Functions/s2RectIntersection.cpp index b108cc1b64f..d4339b4d601 100644 --- a/src/Functions/s2RectIntersection.cpp +++ b/src/Functions/s2RectIntersection.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } namespace @@ -67,10 +68,45 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo1 = arguments[0].column.get(); - const auto * col_hi1 = arguments[1].column.get(); - const auto * col_lo2 = arguments[2].column.get(); - const auto * col_hi2 = arguments[3].column.get(); + const auto * col_lo1 = checkAndGetColumn(arguments[0].column.get()); + if (!col_lo1) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_lo1 = col_lo1->getData(); + + const auto * col_hi1 = checkAndGetColumn(arguments[1].column.get()); + if (!col_hi1) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_hi1 = col_hi1->getData(); + + const auto * col_lo2 = checkAndGetColumn(arguments[2].column.get()); + if (!col_lo2) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_lo2 = col_lo2->getData(); + + const auto * col_hi2 = checkAndGetColumn(arguments[3].column.get()); + if (!col_hi2) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[3].type->getName(), + 4, + getName()); + const auto & data_hi2 = col_hi2->getData(); auto col_res_first = ColumnUInt64::create(); auto col_res_second = ColumnUInt64::create(); @@ -83,10 +119,10 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const auto lo1 = S2CellId(col_lo1->getUInt(row)); - const auto hi1 = S2CellId(col_hi1->getUInt(row)); - const auto lo2 = S2CellId(col_lo2->getUInt(row)); - const auto hi2 = S2CellId(col_hi2->getUInt(row)); + const auto lo1 = S2CellId(data_lo1[row]); + const auto hi1 = S2CellId(data_hi1[row]); + const auto lo2 = S2CellId(data_lo2[row]); + const auto hi2 = S2CellId(data_hi2[row]); if (!lo1.is_valid() || !hi1.is_valid()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "First rectangle is not valid"); diff --git a/src/Functions/s2RectUnion.cpp b/src/Functions/s2RectUnion.cpp index bd40a747a09..047d331e711 100644 --- a/src/Functions/s2RectUnion.cpp +++ b/src/Functions/s2RectUnion.cpp @@ -19,6 +19,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } namespace @@ -65,10 +66,45 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_lo1 = arguments[0].column.get(); - const auto * col_hi1 = arguments[1].column.get(); - const auto * col_lo2 = arguments[2].column.get(); - const auto * col_hi2 = arguments[3].column.get(); + const auto * col_lo1 = checkAndGetColumn(arguments[0].column.get()); + if (!col_lo1) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + const auto & data_lo1 = col_lo1->getData(); + + const auto * col_hi1 = checkAndGetColumn(arguments[1].column.get()); + if (!col_hi1) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[1].type->getName(), + 2, + getName()); + const auto & data_hi1 = col_hi1->getData(); + + const auto * col_lo2 = checkAndGetColumn(arguments[2].column.get()); + if (!col_lo2) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[2].type->getName(), + 3, + getName()); + const auto & data_lo2 = col_lo2->getData(); + + const auto * col_hi2 = checkAndGetColumn(arguments[3].column.get()); + if (!col_hi2) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[3].type->getName(), + 4, + getName()); + const auto & data_hi2 = col_hi2->getData(); auto col_res_first = ColumnUInt64::create(); auto col_res_second = ColumnUInt64::create(); @@ -81,10 +117,10 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const auto lo1 = S2CellId(col_lo1->getUInt(row)); - const auto hi1 = S2CellId(col_hi1->getUInt(row)); - const auto lo2 = S2CellId(col_lo2->getUInt(row)); - const auto hi2 = S2CellId(col_hi2->getUInt(row)); + const auto lo1 = S2CellId(data_lo1[row]); + const auto hi1 = S2CellId(data_hi1[row]); + const auto lo2 = S2CellId(data_lo2[row]); + const auto hi2 = S2CellId(data_hi2[row]); if (!lo1.is_valid() || !hi1.is_valid()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "First rectangle is not valid"); diff --git a/src/Functions/s2ToGeo.cpp b/src/Functions/s2ToGeo.cpp index 03a67d49e45..3d11c21a353 100644 --- a/src/Functions/s2ToGeo.cpp +++ b/src/Functions/s2ToGeo.cpp @@ -21,6 +21,7 @@ namespace ErrorCodes { extern const int ILLEGAL_TYPE_OF_ARGUMENT; extern const int BAD_ARGUMENTS; + extern const int ILLEGAL_COLUMN; } namespace @@ -57,7 +58,7 @@ public: if (!WhichDataType(arg).isUInt64()) throw Exception( ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Illegal type {} of argument {} of function {}. Must be Float64", + "Illegal type {} of argument {} of function {}. Must be UInt64", arg->getName(), 1, getName()); DataTypePtr element = std::make_shared(); @@ -67,7 +68,16 @@ public: ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override { - const auto * col_id = arguments[0].column.get(); + const auto * col_id = checkAndGetColumn(arguments[0].column.get()); + if (!col_id) + throw Exception( + ErrorCodes::ILLEGAL_COLUMN, + "Illegal type {} of argument {} of function {}. Must be UInt64", + arguments[0].type->getName(), + 1, + getName()); + + const auto & data_id = col_id->getData(); auto col_longitude = ColumnFloat64::create(); auto col_latitude = ColumnFloat64::create(); @@ -80,7 +90,7 @@ public: for (size_t row = 0; row < input_rows_count; ++row) { - const auto id = S2CellId(col_id->getUInt(row)); + const auto id = S2CellId(data_id[row]); if (!id.is_valid()) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Point is not valid"); diff --git a/src/IO/Archives/IArchiveReader.h b/src/IO/Archives/IArchiveReader.h new file mode 100644 index 00000000000..584e80a7d09 --- /dev/null +++ b/src/IO/Archives/IArchiveReader.h @@ -0,0 +1,60 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class ReadBuffer; +class ReadBufferFromFileBase; +class SeekableReadBuffer; + +/// Interface for reading an archive. +class IArchiveReader : public std::enable_shared_from_this +{ +public: + virtual ~IArchiveReader() = default; + + /// Returns true if there is a specified file in the archive. + virtual bool fileExists(const String & filename) = 0; + + struct FileInfo + { + UInt64 uncompressed_size; + UInt64 compressed_size; + int compression_method; + bool is_encrypted; + }; + + /// Returns the information about a file stored in the archive. + virtual FileInfo getFileInfo(const String & filename) = 0; + + class FileEnumerator + { + public: + virtual ~FileEnumerator() = default; + virtual const String & getFileName() const = 0; + virtual const FileInfo & getFileInfo() const = 0; + virtual bool nextFile() = 0; + }; + + /// Starts enumerating files in the archive. + virtual std::unique_ptr firstFile() = 0; + + /// Starts reading a file from the archive. The function returns a read buffer, + /// you can read that buffer to extract uncompressed data from the archive. + /// Several read buffers can be used at the same time in parallel. + virtual std::unique_ptr readFile(const String & filename) = 0; + + /// It's possible to convert a file enumerator to a read buffer and vice versa. + virtual std::unique_ptr readFile(std::unique_ptr enumerator) = 0; + virtual std::unique_ptr nextFile(std::unique_ptr read_buffer) = 0; + + /// Sets password used to decrypt files in the archive. + virtual void setPassword(const String & /* password */) {} + + using ReadArchiveFunction = std::function()>; +}; + +} diff --git a/src/IO/Archives/IArchiveWriter.h b/src/IO/Archives/IArchiveWriter.h new file mode 100644 index 00000000000..6879d470b62 --- /dev/null +++ b/src/IO/Archives/IArchiveWriter.h @@ -0,0 +1,38 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class WriteBufferFromFileBase; + +/// Interface for writing an archive. +class IArchiveWriter : public std::enable_shared_from_this +{ +public: + /// Destructors finalizes writing the archive. + virtual ~IArchiveWriter() = default; + + /// Starts writing a file to the archive. The function returns a write buffer, + /// any data written to that buffer will be compressed and then put to the archive. + /// You can keep only one such buffer at a time, a buffer returned by previous call + /// of the function `writeFile()` should be destroyed before next call of `writeFile()`. + virtual std::unique_ptr writeFile(const String & filename) = 0; + + /// Returns true if there is an active instance of WriteBuffer returned by writeFile(). + /// This function should be used mostly for debugging purposes. + virtual bool isWritingFile() const = 0; + + static constexpr const int kDefaultCompressionLevel = -1; + + /// Sets compression method and level. + /// Changing them will affect next file in the archive. + virtual void setCompression(int /* compression_method */, int /* compression_level */ = kDefaultCompressionLevel) {} + + /// Sets password. If the password is not empty it will enable encryption in the archive. + virtual void setPassword(const String & /* password */) {} +}; + +} diff --git a/src/IO/Archives/ZipArchiveReader.cpp b/src/IO/Archives/ZipArchiveReader.cpp new file mode 100644 index 00000000000..16604da62dc --- /dev/null +++ b/src/IO/Archives/ZipArchiveReader.cpp @@ -0,0 +1,563 @@ +#include + +#if USE_MINIZIP +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_UNPACK_ARCHIVE; + extern const int LOGICAL_ERROR; + extern const int SEEK_POSITION_OUT_OF_BOUND; +} + +using RawHandle = unzFile; + + +/// Holds a raw handle, calls acquireRawHandle() in the constructor and releaseRawHandle() in the destructor. +class ZipArchiveReader::HandleHolder +{ +public: + HandleHolder() = default; + + explicit HandleHolder(const std::shared_ptr & reader_) : reader(reader_), raw_handle(reader->acquireRawHandle()) { } + + ~HandleHolder() + { + if (raw_handle) + { + try + { + closeFile(); + } + catch (...) + { + tryLogCurrentException("ZipArchiveReader"); + } + reader->releaseRawHandle(raw_handle); + } + } + + HandleHolder(HandleHolder && src) + { + *this = std::move(src); + } + + HandleHolder & operator =(HandleHolder && src) + { + reader = std::exchange(src.reader, nullptr); + raw_handle = std::exchange(src.raw_handle, nullptr); + file_name = std::exchange(src.file_name, {}); + file_info = std::exchange(src.file_info, {}); + return *this; + } + + RawHandle getRawHandle() const { return raw_handle; } + std::shared_ptr getReader() const { return reader; } + + void locateFile(const String & file_name_) + { + resetFileInfo(); + bool case_sensitive = true; + int err = unzLocateFile(raw_handle, file_name_.c_str(), reinterpret_cast(static_cast(case_sensitive))); + if (err == UNZ_END_OF_LIST_OF_FILE) + showError("File " + quoteString(file_name_) + " not found"); + file_name = file_name_; + } + + bool tryLocateFile(const String & file_name_) + { + resetFileInfo(); + bool case_sensitive = true; + int err = unzLocateFile(raw_handle, file_name_.c_str(), reinterpret_cast(static_cast(case_sensitive))); + if (err == UNZ_END_OF_LIST_OF_FILE) + return false; + checkResult(err); + file_name = file_name_; + return true; + } + + bool firstFile() + { + resetFileInfo(); + int err = unzGoToFirstFile(raw_handle); + if (err == UNZ_END_OF_LIST_OF_FILE) + return false; + checkResult(err); + return true; + } + + bool nextFile() + { + resetFileInfo(); + int err = unzGoToNextFile(raw_handle); + if (err == UNZ_END_OF_LIST_OF_FILE) + return false; + checkResult(err); + return true; + } + + const String & getFileName() const + { + if (!file_name) + retrieveFileInfo(); + return *file_name; + } + + const FileInfo & getFileInfo() const + { + if (!file_info) + retrieveFileInfo(); + return *file_info; + } + + void closeFile() + { + int err = unzCloseCurrentFile(raw_handle); + /// If err == UNZ_PARAMERROR the file is already closed. + if (err != UNZ_PARAMERROR) + checkResult(err); + } + + void checkResult(int code) const { reader->checkResult(code); } + [[noreturn]] void showError(const String & message) const { reader->showError(message); } + +private: + void retrieveFileInfo() const + { + if (file_name && file_info) + return; + unz_file_info64 finfo; + int err = unzGetCurrentFileInfo64(raw_handle, &finfo, nullptr, 0, nullptr, 0, nullptr, 0); + if (err == UNZ_PARAMERROR) + showError("No current file"); + checkResult(err); + if (!file_info) + { + file_info.emplace(); + file_info->uncompressed_size = finfo.uncompressed_size; + file_info->compressed_size = finfo.compressed_size; + file_info->compression_method = finfo.compression_method; + file_info->is_encrypted = (finfo.flag & MZ_ZIP_FLAG_ENCRYPTED); + } + if (!file_name) + { + file_name.emplace(); + file_name->resize(finfo.size_filename); + checkResult(unzGetCurrentFileInfo64(raw_handle, nullptr, file_name->data(), finfo.size_filename, nullptr, 0, nullptr, 0)); + } + } + + void resetFileInfo() + { + file_info.reset(); + file_name.reset(); + } + + std::shared_ptr reader; + RawHandle raw_handle = nullptr; + mutable std::optional file_name; + mutable std::optional file_info; +}; + + +/// This class represents a ReadBuffer actually returned by readFile(). +class ZipArchiveReader::ReadBufferFromZipArchive : public ReadBufferFromFileBase +{ +public: + explicit ReadBufferFromZipArchive(HandleHolder && handle_) + : ReadBufferFromFileBase(DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0) + , handle(std::move(handle_)) + { + const auto & file_info = handle.getFileInfo(); + checkCompressionMethodIsEnabled(static_cast(file_info.compression_method)); + + const char * password_cstr = nullptr; + if (file_info.is_encrypted) + { + const auto & password_str = handle.getReader()->password; + if (password_str.empty()) + showError("Password is required"); + password_cstr = password_str.c_str(); + checkEncryptionIsEnabled(); + } + + RawHandle raw_handle = handle.getRawHandle(); + int err = unzOpenCurrentFilePassword(raw_handle, password_cstr); + if (err == MZ_PASSWORD_ERROR) + showError("Wrong password"); + checkResult(err); + } + + off_t seek(off_t off, int whence) override + { + off_t current_pos = getPosition(); + off_t new_pos; + if (whence == SEEK_SET) + new_pos = off; + else if (whence == SEEK_CUR) + new_pos = off + current_pos; + else + throw Exception("Only SEEK_SET and SEEK_CUR seek modes allowed.", ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); + + if (new_pos == current_pos) + return current_pos; /// The position is the same. + + if (new_pos < 0) + throw Exception("Seek position is out of bound", ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); + + off_t working_buffer_start_pos = current_pos - offset(); + off_t working_buffer_end_pos = current_pos + available(); + + if ((working_buffer_start_pos <= new_pos) && (new_pos <= working_buffer_end_pos)) + { + /// The new position is still inside the buffer. + position() += new_pos - current_pos; + return new_pos; + } + + RawHandle raw_handle = handle.getRawHandle(); + + /// Check that the new position is now beyond the end of the file. + const auto & file_info = handle.getFileInfo(); + if (new_pos > static_cast(file_info.uncompressed_size)) + throw Exception("Seek position is out of bound", ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); + + if (file_info.compression_method == static_cast(CompressionMethod::kStore)) + { + /// unzSeek64() works only for non-compressed files. + checkResult(unzSeek64(raw_handle, off, whence)); + return unzTell64(raw_handle); + } + + /// As a last try we go slow way, we're going to simply ignore all data before the new position. + if (new_pos < current_pos) + { + checkResult(unzCloseCurrentFile(raw_handle)); + checkResult(unzOpenCurrentFile(raw_handle)); + current_pos = 0; + } + + ignore(new_pos - current_pos); + return new_pos; + } + + off_t getPosition() override + { + RawHandle raw_handle = handle.getRawHandle(); + return unzTell64(raw_handle) - available(); + } + + String getFileName() const override { return handle.getFileName(); } + + /// Releases owned handle to pass it to an enumerator. + HandleHolder releaseHandle() && + { + handle.closeFile(); + return std::move(handle); + } + +private: + bool nextImpl() override + { + RawHandle raw_handle = handle.getRawHandle(); + auto bytes_read = unzReadCurrentFile(raw_handle, internal_buffer.begin(), internal_buffer.size()); + + if (bytes_read < 0) + checkResult(bytes_read); + + if (!bytes_read) + return false; + + working_buffer = internal_buffer; + working_buffer.resize(bytes_read); + return true; + } + + void checkResult(int code) const { handle.checkResult(code); } + [[noreturn]] void showError(const String & message) const { handle.showError(message); } + + HandleHolder handle; +}; + + +class ZipArchiveReader::FileEnumeratorImpl : public FileEnumerator +{ +public: + explicit FileEnumeratorImpl(HandleHolder && handle_) : handle(std::move(handle_)) {} + + const String & getFileName() const override { return handle.getFileName(); } + const FileInfo & getFileInfo() const override { return handle.getFileInfo(); } + bool nextFile() override { return handle.nextFile(); } + + /// Releases owned handle to pass it to a read buffer. + HandleHolder releaseHandle() && { return std::move(handle); } + +private: + HandleHolder handle; +}; + + +namespace +{ + /// Provides a set of functions allowing the minizip library to read its input + /// from a SeekableReadBuffer instead of an ordinary file in the local filesystem. + class StreamFromReadBuffer + { + public: + static RawHandle open(std::unique_ptr archive_read_buffer, UInt64 archive_size) + { + StreamFromReadBuffer::Opaque opaque{std::move(archive_read_buffer), archive_size}; + + zlib_filefunc64_def func_def; + func_def.zopen64_file = &StreamFromReadBuffer::openFileFunc; + func_def.zclose_file = &StreamFromReadBuffer::closeFileFunc; + func_def.zread_file = &StreamFromReadBuffer::readFileFunc; + func_def.zwrite_file = &StreamFromReadBuffer::writeFileFunc; + func_def.zseek64_file = &StreamFromReadBuffer::seekFunc; + func_def.ztell64_file = &StreamFromReadBuffer::tellFunc; + func_def.zerror_file = &StreamFromReadBuffer::testErrorFunc; + func_def.opaque = &opaque; + + return unzOpen2_64(/* path= */ nullptr, + &func_def); + } + + private: + std::unique_ptr read_buffer; + UInt64 start_offset = 0; + UInt64 total_size = 0; + bool at_end = false; + + struct Opaque + { + std::unique_ptr read_buffer; + UInt64 total_size = 0; + }; + + static void * openFileFunc(void * opaque, const void *, int) + { + auto & opq = *reinterpret_cast(opaque); + return new StreamFromReadBuffer(std::move(opq.read_buffer), opq.total_size); + } + + StreamFromReadBuffer(std::unique_ptr read_buffer_, UInt64 total_size_) + : read_buffer(std::move(read_buffer_)), start_offset(read_buffer->getPosition()), total_size(total_size_) {} + + static int closeFileFunc(void *, void * stream) + { + delete reinterpret_cast(stream); + return ZIP_OK; + } + + static StreamFromReadBuffer & get(void * ptr) + { + return *reinterpret_cast(ptr); + } + + static int testErrorFunc(void *, void *) + { + return ZIP_OK; + } + + static unsigned long readFileFunc(void *, void * stream, void * buf, unsigned long size) // NOLINT(google-runtime-int) + { + auto & strm = get(stream); + if (strm.at_end) + return 0; + auto read_bytes = strm.read_buffer->read(reinterpret_cast(buf), size); + return read_bytes; + } + + static ZPOS64_T tellFunc(void *, void * stream) + { + auto & strm = get(stream); + if (strm.at_end) + return strm.total_size; + auto pos = strm.read_buffer->getPosition() - strm.start_offset; + return pos; + } + + static long seekFunc(void *, void * stream, ZPOS64_T offset, int origin) // NOLINT(google-runtime-int) + { + auto & strm = get(stream); + if (origin == SEEK_END) + { + /// Our implementations of SeekableReadBuffer don't support SEEK_END, + /// but the minizip library needs it, so we have to simulate it here. + strm.at_end = true; + return ZIP_OK; + } + strm.at_end = false; + if (origin == SEEK_SET) + offset += strm.start_offset; + strm.read_buffer->seek(offset, origin); + return ZIP_OK; + } + + static unsigned long writeFileFunc(void *, void *, const void *, unsigned long) // NOLINT(google-runtime-int) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "StreamFromReadBuffer::writeFile must not be called"); + } + }; +} + + +ZipArchiveReader::ZipArchiveReader(const String & path_to_archive_) + : path_to_archive(path_to_archive_) +{ + init(); + +} + +ZipArchiveReader::ZipArchiveReader( + const String & path_to_archive_, const ReadArchiveFunction & archive_read_function_, UInt64 archive_size_) + : path_to_archive(path_to_archive_), archive_read_function(archive_read_function_), archive_size(archive_size_) +{ + init(); +} + +void ZipArchiveReader::init() +{ + /// Prepare the first handle in `free_handles` and check that the archive can be read. + releaseRawHandle(acquireRawHandle()); +} + +ZipArchiveReader::~ZipArchiveReader() +{ + /// Close all `free_handles`. + for (RawHandle free_handle : free_handles) + { + try + { + checkResult(unzClose(free_handle)); + } + catch (...) + { + tryLogCurrentException("ZipArchiveReader"); + } + } +} + +bool ZipArchiveReader::fileExists(const String & filename) +{ + return acquireHandle().tryLocateFile(filename); +} + +ZipArchiveReader::FileInfo ZipArchiveReader::getFileInfo(const String & filename) +{ + auto handle = acquireHandle(); + handle.locateFile(filename); + return handle.getFileInfo(); +} + +std::unique_ptr ZipArchiveReader::firstFile() +{ + auto handle = acquireHandle(); + if (!handle.firstFile()) + return nullptr; + return std::make_unique(std::move(handle)); +} + +std::unique_ptr ZipArchiveReader::readFile(const String & filename) +{ + auto handle = acquireHandle(); + handle.locateFile(filename); + return std::make_unique(std::move(handle)); +} + +std::unique_ptr ZipArchiveReader::readFile(std::unique_ptr enumerator) +{ + if (!dynamic_cast(enumerator.get())) + throw Exception("Wrong enumerator passed to readFile()", ErrorCodes::LOGICAL_ERROR); + auto enumerator_impl = std::unique_ptr(static_cast(enumerator.release())); + auto handle = std::move(*enumerator_impl).releaseHandle(); + return std::make_unique(std::move(handle)); +} + +std::unique_ptr ZipArchiveReader::nextFile(std::unique_ptr read_buffer) +{ + if (!dynamic_cast(read_buffer.get())) + throw Exception("Wrong ReadBuffer passed to nextFile()", ErrorCodes::LOGICAL_ERROR); + auto read_buffer_from_zip = std::unique_ptr(static_cast(read_buffer.release())); + auto handle = std::move(*read_buffer_from_zip).releaseHandle(); + if (!handle.nextFile()) + return nullptr; + return std::make_unique(std::move(handle)); +} + +void ZipArchiveReader::setPassword(const String & password_) +{ + std::lock_guard lock{mutex}; + password = password_; +} + +ZipArchiveReader::HandleHolder ZipArchiveReader::acquireHandle() +{ + return HandleHolder{std::static_pointer_cast(shared_from_this())}; +} + +ZipArchiveReader::RawHandle ZipArchiveReader::acquireRawHandle() +{ + std::lock_guard lock{mutex}; + + if (!free_handles.empty()) + { + RawHandle free_handle = free_handles.back(); + free_handles.pop_back(); + return free_handle; + } + + RawHandle new_handle = nullptr; + if (archive_read_function) + new_handle = StreamFromReadBuffer::open(archive_read_function(), archive_size); + else + new_handle = unzOpen64(path_to_archive.c_str()); + + if (!new_handle) + throw Exception(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "Couldn't open zip archive {}", quoteString(path_to_archive)); + + return new_handle; +} + +void ZipArchiveReader::releaseRawHandle(RawHandle handle_) +{ + if (!handle_) + return; + + std::lock_guard lock{mutex}; + free_handles.push_back(handle_); +} + +void ZipArchiveReader::checkResult(int code) const +{ + if (code >= UNZ_OK) + return; + + String message = "Code= "; + switch (code) + { + case UNZ_OK: return; + case UNZ_ERRNO: message += "ERRNO, errno= " + String{strerror(errno)}; break; + case UNZ_PARAMERROR: message += "PARAMERROR"; break; + case UNZ_BADZIPFILE: message += "BADZIPFILE"; break; + case UNZ_INTERNALERROR: message += "INTERNALERROR"; break; + case UNZ_CRCERROR: message += "CRCERROR"; break; + case UNZ_BADPASSWORD: message += "BADPASSWORD"; break; + default: message += std::to_string(code); break; + } + showError(message); +} + +void ZipArchiveReader::showError(const String & message) const +{ + throw Exception(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "Couldn't unpack zip archive {}: {}", quoteString(path_to_archive), message); +} + +} + +#endif diff --git a/src/IO/Archives/ZipArchiveReader.h b/src/IO/Archives/ZipArchiveReader.h new file mode 100644 index 00000000000..6932a93e23f --- /dev/null +++ b/src/IO/Archives/ZipArchiveReader.h @@ -0,0 +1,86 @@ +#pragma once + +#include + +#if USE_MINIZIP +#include +#include +#include +#include +#include + + +namespace DB +{ +class ReadBuffer; +class ReadBufferFromFileBase; +class SeekableReadBuffer; + +/// Implementation of IArchiveReader for reading zip archives. +class ZipArchiveReader : public shared_ptr_helper, public IArchiveReader +{ +public: + using CompressionMethod = ZipArchiveWriter::CompressionMethod; + + ~ZipArchiveReader() override; + + /// Returns true if there is a specified file in the archive. + bool fileExists(const String & filename) override; + + /// Returns the information about a file stored in the archive. + FileInfo getFileInfo(const String & filename) override; + + /// Starts enumerating files in the archive. + std::unique_ptr firstFile() override; + + /// Starts reading a file from the archive. The function returns a read buffer, + /// you can read that buffer to extract uncompressed data from the archive. + /// Several read buffers can be used at the same time in parallel. + std::unique_ptr readFile(const String & filename) override; + + /// It's possible to convert a file enumerator to a read buffer and vice versa. + std::unique_ptr readFile(std::unique_ptr enumerator) override; + std::unique_ptr nextFile(std::unique_ptr read_buffer) override; + + /// Sets password used to decrypt the contents of the files in the archive. + void setPassword(const String & password_) override; + + /// Utility functions. + static CompressionMethod parseCompressionMethod(const String & str) { return ZipArchiveWriter::parseCompressionMethod(str); } + static void checkCompressionMethodIsEnabled(CompressionMethod method) { ZipArchiveWriter::checkCompressionMethodIsEnabled(method); } + static void checkEncryptionIsEnabled() { ZipArchiveWriter::checkEncryptionIsEnabled(); } + +private: + /// Constructs an archive's reader that will read from a file in the local filesystem. + explicit ZipArchiveReader(const String & path_to_archive_); + + /// Constructs an archive's reader that will read by making a read buffer by using + /// a specified function. + ZipArchiveReader(const String & path_to_archive_, const ReadArchiveFunction & archive_read_function_, UInt64 archive_size_); + + friend struct shared_ptr_helper; + class ReadBufferFromZipArchive; + class FileEnumeratorImpl; + class HandleHolder; + using RawHandle = void *; + + void init(); + + HandleHolder acquireHandle(); + RawHandle acquireRawHandle(); + void releaseRawHandle(RawHandle handle_); + + void checkResult(int code) const; + [[noreturn]] void showError(const String & message) const; + + const String path_to_archive; + const ReadArchiveFunction archive_read_function; + const UInt64 archive_size = 0; + String password; + std::vector free_handles; + mutable std::mutex mutex; +}; + +} + +#endif diff --git a/src/IO/Archives/ZipArchiveWriter.cpp b/src/IO/Archives/ZipArchiveWriter.cpp new file mode 100644 index 00000000000..f5ecea5e5aa --- /dev/null +++ b/src/IO/Archives/ZipArchiveWriter.cpp @@ -0,0 +1,385 @@ +#include + +#if USE_MINIZIP +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_PACK_ARCHIVE; + extern const int SUPPORT_IS_DISABLED; + extern const int LOGICAL_ERROR; +} + +using RawHandle = zipFile; + + +/// Holds a raw handle, calls acquireRawHandle() in the constructor and releaseRawHandle() in the destructor. +class ZipArchiveWriter::HandleHolder +{ +public: + HandleHolder() = default; + + explicit HandleHolder(const std::shared_ptr & writer_) : writer(writer_), raw_handle(writer->acquireRawHandle()) { } + + ~HandleHolder() + { + if (raw_handle) + { + try + { + int err = zipCloseFileInZip(raw_handle); + /// If err == ZIP_PARAMERROR the file is already closed. + if (err != ZIP_PARAMERROR) + checkResult(err); + } + catch (...) + { + tryLogCurrentException("ZipArchiveWriter"); + } + writer->releaseRawHandle(raw_handle); + } + } + + HandleHolder(HandleHolder && src) + { + *this = std::move(src); + } + + HandleHolder & operator =(HandleHolder && src) + { + writer = std::exchange(src.writer, nullptr); + raw_handle = std::exchange(src.raw_handle, nullptr); + return *this; + } + + RawHandle getRawHandle() const { return raw_handle; } + std::shared_ptr getWriter() const { return writer; } + + void checkResult(int code) const { writer->checkResult(code); } + +private: + std::shared_ptr writer; + RawHandle raw_handle = nullptr; +}; + + +/// This class represents a WriteBuffer actually returned by writeFile(). +class ZipArchiveWriter::WriteBufferFromZipArchive : public WriteBufferFromFileBase +{ +public: + WriteBufferFromZipArchive(HandleHolder && handle_, const String & filename_) + : WriteBufferFromFileBase(DBMS_DEFAULT_BUFFER_SIZE, nullptr, 0) + , handle(std::move(handle_)) + , filename(filename_) + { + auto compress_method = handle.getWriter()->compression_method; + auto compress_level = handle.getWriter()->compression_level; + checkCompressionMethodIsEnabled(static_cast(compress_method)); + + const char * password_cstr = nullptr; + const String & password_str = handle.getWriter()->password; + if (!password_str.empty()) + { + checkEncryptionIsEnabled(); + password_cstr = password_str.c_str(); + } + + RawHandle raw_handle = handle.getRawHandle(); + + checkResult(zipOpenNewFileInZip3_64( + raw_handle, + filename_.c_str(), + /* zipfi= */ nullptr, + /* extrafield_local= */ nullptr, + /* size_extrafield_local= */ 0, + /* extrafield_global= */ nullptr, + /* size_extrafield_global= */ 0, + /* comment= */ nullptr, + compress_method, + compress_level, + /* raw= */ false, + /* windowBits= */ 0, + /* memLevel= */ 0, + /* strategy= */ 0, + password_cstr, + /* crc_for_crypting= */ 0, + /* zip64= */ true)); + } + + ~WriteBufferFromZipArchive() override + { + try + { + finalize(); + } + catch (...) + { + tryLogCurrentException("ZipArchiveWriter"); + } + } + + void sync() override { next(); } + std::string getFileName() const override { return filename; } + +private: + void nextImpl() override + { + if (!offset()) + return; + RawHandle raw_handle = handle.getRawHandle(); + checkResult(zipWriteInFileInZip(raw_handle, working_buffer.begin(), offset())); + } + + void checkResult(int code) const { handle.checkResult(code); } + + HandleHolder handle; + String filename; +}; + + +namespace +{ + /// Provides a set of functions allowing the minizip library to write its output + /// to a WriteBuffer instead of an ordinary file in the local filesystem. + class StreamFromWriteBuffer + { + public: + static RawHandle open(std::unique_ptr archive_write_buffer) + { + Opaque opaque{std::move(archive_write_buffer)}; + + zlib_filefunc64_def func_def; + func_def.zopen64_file = &StreamFromWriteBuffer::openFileFunc; + func_def.zclose_file = &StreamFromWriteBuffer::closeFileFunc; + func_def.zread_file = &StreamFromWriteBuffer::readFileFunc; + func_def.zwrite_file = &StreamFromWriteBuffer::writeFileFunc; + func_def.zseek64_file = &StreamFromWriteBuffer::seekFunc; + func_def.ztell64_file = &StreamFromWriteBuffer::tellFunc; + func_def.zerror_file = &StreamFromWriteBuffer::testErrorFunc; + func_def.opaque = &opaque; + + return zipOpen2_64( + /* path= */ nullptr, + /* append= */ false, + /* globalcomment= */ nullptr, + &func_def); + } + + private: + std::unique_ptr write_buffer; + UInt64 start_offset = 0; + + struct Opaque + { + std::unique_ptr write_buffer; + }; + + static void * openFileFunc(void * opaque, const void *, int) + { + Opaque & opq = *reinterpret_cast(opaque); + return new StreamFromWriteBuffer(std::move(opq.write_buffer)); + } + + explicit StreamFromWriteBuffer(std::unique_ptr write_buffer_) + : write_buffer(std::move(write_buffer_)), start_offset(write_buffer->count()) {} + + static int closeFileFunc(void *, void * stream) + { + delete reinterpret_cast(stream); + return ZIP_OK; + } + + static StreamFromWriteBuffer & get(void * ptr) + { + return *reinterpret_cast(ptr); + } + + static unsigned long writeFileFunc(void *, void * stream, const void * buf, unsigned long size) // NOLINT(google-runtime-int) + { + auto & strm = get(stream); + strm.write_buffer->write(reinterpret_cast(buf), size); + return size; + } + + static int testErrorFunc(void *, void *) + { + return ZIP_OK; + } + + static ZPOS64_T tellFunc(void *, void * stream) + { + auto & strm = get(stream); + auto pos = strm.write_buffer->count() - strm.start_offset; + return pos; + } + + static long seekFunc(void *, void *, ZPOS64_T, int) // NOLINT(google-runtime-int) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "StreamFromWriteBuffer::seek must not be called"); + } + + static unsigned long readFileFunc(void *, void *, void *, unsigned long) // NOLINT(google-runtime-int) + { + throw Exception(ErrorCodes::LOGICAL_ERROR, "StreamFromWriteBuffer::readFile must not be called"); + } + }; +} + + +ZipArchiveWriter::ZipArchiveWriter(const String & path_to_archive_) + : ZipArchiveWriter(path_to_archive_, nullptr) +{ +} + +ZipArchiveWriter::ZipArchiveWriter(const String & path_to_archive_, std::unique_ptr archive_write_buffer_) + : path_to_archive(path_to_archive_) +{ + if (archive_write_buffer_) + handle = StreamFromWriteBuffer::open(std::move(archive_write_buffer_)); + else + handle = zipOpen64(path_to_archive.c_str(), /* append= */ false); + if (!handle) + throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Couldn't create zip archive {}", quoteString(path_to_archive)); +} + +ZipArchiveWriter::~ZipArchiveWriter() +{ + if (handle) + { + try + { + checkResult(zipClose(handle, /* global_comment= */ nullptr)); + } + catch (...) + { + tryLogCurrentException("ZipArchiveWriter"); + } + } +} + +std::unique_ptr ZipArchiveWriter::writeFile(const String & filename) +{ + return std::make_unique(acquireHandle(), filename); +} + +bool ZipArchiveWriter::isWritingFile() const +{ + std::lock_guard lock{mutex}; + return !handle; +} + +void ZipArchiveWriter::setCompression(int compression_method_, int compression_level_) +{ + std::lock_guard lock{mutex}; + compression_method = compression_method_; + compression_level = compression_level_; +} + +void ZipArchiveWriter::setPassword(const String & password_) +{ + std::lock_guard lock{mutex}; + password = password_; +} + +ZipArchiveWriter::CompressionMethod ZipArchiveWriter::parseCompressionMethod(const String & str) +{ + if (str.empty()) + return CompressionMethod::kDeflate; /// Default compression method is DEFLATE. + else if (boost::iequals(str, "store")) + return CompressionMethod::kStore; + else if (boost::iequals(str, "deflate")) + return CompressionMethod::kDeflate; + else if (boost::iequals(str, "bzip2")) + return CompressionMethod::kBzip2; + else if (boost::iequals(str, "lzma")) + return CompressionMethod::kLzma; + else if (boost::iequals(str, "zstd")) + return CompressionMethod::kZstd; + else if (boost::iequals(str, "xz")) + return CompressionMethod::kXz; + else + throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Unknown compression method specified for a zip archive: {}", str); +} + +/// Checks that a passed compression method can be used. +void ZipArchiveWriter::checkCompressionMethodIsEnabled(CompressionMethod method) +{ + switch (method) + { + case CompressionMethod::kStore: [[fallthrough]]; + case CompressionMethod::kDeflate: + case CompressionMethod::kLzma: + case CompressionMethod::kXz: + case CompressionMethod::kZstd: + return; + + case CompressionMethod::kBzip2: + { +#if USE_BZIP2 + return; +#else + throw Exception("BZIP2 compression method is disabled", ErrorCodes::SUPPORT_IS_DISABLED); +#endif + } + } + throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Unknown compression method specified for a zip archive: {}", static_cast(method)); +} + +/// Checks that encryption is enabled. +void ZipArchiveWriter::checkEncryptionIsEnabled() +{ +#if !USE_SSL + throw Exception("Encryption in zip archive is disabled", ErrorCodes::SUPPORT_IS_DISABLED); +#endif +} + +ZipArchiveWriter::HandleHolder ZipArchiveWriter::acquireHandle() +{ + return HandleHolder{std::static_pointer_cast(shared_from_this())}; +} + +RawHandle ZipArchiveWriter::acquireRawHandle() +{ + std::lock_guard lock{mutex}; + if (!handle) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot have more than one write buffer while writing a zip archive"); + return std::exchange(handle, nullptr); +} + +void ZipArchiveWriter::releaseRawHandle(RawHandle raw_handle_) +{ + std::lock_guard lock{mutex}; + handle = raw_handle_; +} + +void ZipArchiveWriter::checkResult(int code) const +{ + if (code >= ZIP_OK) + return; + + String message = "Code= "; + switch (code) + { + case ZIP_ERRNO: message += "ERRNO, errno= " + String{strerror(errno)}; break; + case ZIP_PARAMERROR: message += "PARAMERROR"; break; + case ZIP_BADZIPFILE: message += "BADZIPFILE"; break; + case ZIP_INTERNALERROR: message += "INTERNALERROR"; break; + default: message += std::to_string(code); break; + } + showError(message); +} + +void ZipArchiveWriter::showError(const String & message) const +{ + throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Couldn't pack zip archive {}: {}", quoteString(path_to_archive), message); +} + +} + +#endif diff --git a/src/IO/Archives/ZipArchiveWriter.h b/src/IO/Archives/ZipArchiveWriter.h new file mode 100644 index 00000000000..76f8dd8e9e5 --- /dev/null +++ b/src/IO/Archives/ZipArchiveWriter.h @@ -0,0 +1,97 @@ +#pragma once + +#include + +#if USE_MINIZIP +#include +#include +#include + + +namespace DB +{ +class WriteBuffer; +class WriteBufferFromFileBase; + +/// Implementation of IArchiveWriter for writing zip archives. +class ZipArchiveWriter : public shared_ptr_helper, public IArchiveWriter +{ +public: + /// Destructors finalizes writing the archive. + ~ZipArchiveWriter() override; + + /// Starts writing a file to the archive. The function returns a write buffer, + /// any data written to that buffer will be compressed and then put to the archive. + /// You can keep only one such buffer at a time, a buffer returned by previous call + /// of the function `writeFile()` should be destroyed before next call of `writeFile()`. + std::unique_ptr writeFile(const String & filename) override; + + /// Returns true if there is an active instance of WriteBuffer returned by writeFile(). + /// This function should be used mostly for debugging purposes. + bool isWritingFile() const override; + + /// Supported compression methods. + enum class CompressionMethod + { + /// See mz.h + kStore = 0, + kDeflate = 8, + kBzip2 = 12, + kLzma = 14, + kZstd = 93, + kXz = 95, + }; + + /// Some compression levels. + enum class CompressionLevels + { + kDefault = kDefaultCompressionLevel, + kFast = 2, + kNormal = 6, + kBest = 9, + }; + + /// Sets compression method and level. + /// Changing them will affect next file in the archive. + void setCompression(int compression_method_, int compression_level_) override; + + /// Sets password. Only contents of the files are encrypted, + /// names of files are not encrypted. + /// Changing the password will affect next file in the archive. + void setPassword(const String & password_) override; + + /// Utility functions. + static CompressionMethod parseCompressionMethod(const String & str); + static void checkCompressionMethodIsEnabled(CompressionMethod method); + static void checkEncryptionIsEnabled(); + +private: + /// Constructs an archive that will be written as a file in the local filesystem. + explicit ZipArchiveWriter(const String & path_to_archive_); + + /// Constructs an archive that will be written by using a specified `archive_write_buffer_`. + ZipArchiveWriter(const String & path_to_archive_, std::unique_ptr archive_write_buffer_); + + friend struct shared_ptr_helper; + class WriteBufferFromZipArchive; + class HandleHolder; + using RawHandle = void *; + + HandleHolder acquireHandle(); + RawHandle acquireRawHandle(); + void releaseRawHandle(RawHandle raw_handle_); + + void checkResult(int code) const; + [[noreturn]] void showError(const String & message) const; + + const String path_to_archive; + int compression_method = static_cast(CompressionMethod::kDeflate); + int compression_level = kDefaultCompressionLevel; + String password; + RawHandle handle = nullptr; + mutable std::mutex mutex; +}; + +} + +#endif diff --git a/src/IO/Archives/createArchiveReader.cpp b/src/IO/Archives/createArchiveReader.cpp new file mode 100644 index 00000000000..6ebab000a18 --- /dev/null +++ b/src/IO/Archives/createArchiveReader.cpp @@ -0,0 +1,38 @@ +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_UNPACK_ARCHIVE; + extern const int SUPPORT_IS_DISABLED; +} + + +std::shared_ptr createArchiveReader(const String & path_to_archive) +{ + return createArchiveReader(path_to_archive, {}, 0); +} + + +std::shared_ptr createArchiveReader( + const String & path_to_archive, + [[maybe_unused]] const std::function()> & archive_read_function, + [[maybe_unused]] size_t archive_size) +{ + if (path_to_archive.ends_with(".zip") || path_to_archive.ends_with(".zipx")) + { +#if USE_MINIZIP + return ZipArchiveReader::create(path_to_archive, archive_read_function, archive_size); +#else + throw Exception("minizip library is disabled", ErrorCodes::SUPPORT_IS_DISABLED); +#endif + } + else + throw Exception(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "Cannot determine the type of archive {}", path_to_archive); +} + +} diff --git a/src/IO/Archives/createArchiveReader.h b/src/IO/Archives/createArchiveReader.h new file mode 100644 index 00000000000..9e1073b9481 --- /dev/null +++ b/src/IO/Archives/createArchiveReader.h @@ -0,0 +1,22 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class IArchiveReader; +class SeekableReadBuffer; + +/// Starts reading a specified archive in the local filesystem. +std::shared_ptr createArchiveReader(const String & path_to_archive); + +/// Starts reading a specified archive, the archive is read by using a specified read buffer, +/// `path_to_archive` is used only to determine the archive's type. +std::shared_ptr createArchiveReader( + const String & path_to_archive, + const std::function()> & archive_read_function, + size_t archive_size); + +} diff --git a/src/IO/Archives/createArchiveWriter.cpp b/src/IO/Archives/createArchiveWriter.cpp new file mode 100644 index 00000000000..26cbde8c363 --- /dev/null +++ b/src/IO/Archives/createArchiveWriter.cpp @@ -0,0 +1,38 @@ +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int CANNOT_PACK_ARCHIVE; + extern const int SUPPORT_IS_DISABLED; +} + + +std::shared_ptr createArchiveWriter(const String & path_to_archive) +{ + return createArchiveWriter(path_to_archive, nullptr); +} + + +std::shared_ptr createArchiveWriter( + const String & path_to_archive, + [[maybe_unused]] std::unique_ptr archive_write_buffer) +{ + if (path_to_archive.ends_with(".zip") || path_to_archive.ends_with(".zipx")) + { +#if USE_MINIZIP + return ZipArchiveWriter::create(path_to_archive, std::move(archive_write_buffer)); +#else + throw Exception("minizip library is disabled", ErrorCodes::SUPPORT_IS_DISABLED); +#endif + } + else + throw Exception(ErrorCodes::CANNOT_PACK_ARCHIVE, "Cannot determine the type of archive {}", path_to_archive); +} + +} diff --git a/src/IO/Archives/createArchiveWriter.h b/src/IO/Archives/createArchiveWriter.h new file mode 100644 index 00000000000..51ffd4d1144 --- /dev/null +++ b/src/IO/Archives/createArchiveWriter.h @@ -0,0 +1,19 @@ +#pragma once + +#include +#include + + +namespace DB +{ +class IArchiveWriter; +class WriteBuffer; + +/// Starts writing a specified archive in the local filesystem. +std::shared_ptr createArchiveWriter(const String & path_to_archive); + +/// Starts writing a specified archive, the archive is written by using a specified write buffer, +/// `path_to_archive` is used only to determine the archive's type. +std::shared_ptr createArchiveWriter(const String & path_to_archive, std::unique_ptr archive_write_buffer); + +} diff --git a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp index 9c92201b3a1..877702f9705 100644 --- a/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp +++ b/src/IO/AsynchronousReadBufferFromFileDescriptor.cpp @@ -181,8 +181,8 @@ off_t AsynchronousReadBufferFromFileDescriptor::seek(off_t offset, int whence) off_t offset_after_seek_pos = new_pos - seek_pos; - /// First put position at the end of the buffer so the next read will fetch new data to the buffer. - pos = working_buffer.end(); + /// First reset the buffer so the next read will fetch new data to the buffer. + resetWorkingBuffer(); /// Just update the info about the next position in file. diff --git a/src/IO/BufferBase.h b/src/IO/BufferBase.h index 198441d8bc1..7a59687fa56 100644 --- a/src/IO/BufferBase.h +++ b/src/IO/BufferBase.h @@ -97,6 +97,15 @@ public: bool isPadded() const { return padded; } protected: + void resetWorkingBuffer() + { + /// Move position to the end of buffer to trigger call of 'next' on next reading. + /// Discard all data in current working buffer to prevent wrong assumptions on content + /// of buffer, e.g. for optimizations of seeks in seekable buffers. + working_buffer.resize(0); + pos = working_buffer.end(); + } + /// Read/write position. Position pos; diff --git a/src/IO/ReadBufferFromEncryptedFile.cpp b/src/IO/ReadBufferFromEncryptedFile.cpp index 445c55ac269..7aec6dcde02 100644 --- a/src/IO/ReadBufferFromEncryptedFile.cpp +++ b/src/IO/ReadBufferFromEncryptedFile.cpp @@ -56,7 +56,7 @@ off_t ReadBufferFromEncryptedFile::seek(off_t off, int whence) offset = new_pos; /// No more reading from the current working buffer until next() is called. - pos = working_buffer.end(); + resetWorkingBuffer(); assert(!hasPendingData()); } diff --git a/src/IO/ReadBufferFromFileDescriptor.cpp b/src/IO/ReadBufferFromFileDescriptor.cpp index ed6b1a60181..d266fb86e0f 100644 --- a/src/IO/ReadBufferFromFileDescriptor.cpp +++ b/src/IO/ReadBufferFromFileDescriptor.cpp @@ -176,8 +176,8 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence) off_t offset_after_seek_pos = new_pos - seek_pos; - /// First put position at the end of the buffer so the next read will fetch new data to the buffer. - pos = working_buffer.end(); + /// First reset the buffer so the next read will fetch new data to the buffer. + resetWorkingBuffer(); /// In case of using 'pread' we just update the info about the next position in file. /// In case of using 'read' we call 'lseek'. diff --git a/src/IO/ReadBufferFromMemory.cpp b/src/IO/ReadBufferFromMemory.cpp index 98c39c833b0..d0863878797 100644 --- a/src/IO/ReadBufferFromMemory.cpp +++ b/src/IO/ReadBufferFromMemory.cpp @@ -12,31 +12,33 @@ off_t ReadBufferFromMemory::seek(off_t offset, int whence) { if (whence == SEEK_SET) { - if (offset >= 0 && working_buffer.begin() + offset < working_buffer.end()) + if (offset >= 0 && internal_buffer.begin() + offset < internal_buffer.end()) { - pos = working_buffer.begin() + offset; - return size_t(pos - working_buffer.begin()); + pos = internal_buffer.begin() + offset; + working_buffer = internal_buffer; /// We need to restore `working_buffer` in case the position was at EOF before this seek(). + return size_t(pos - internal_buffer.begin()); } else throw Exception( "Seek position is out of bounds. " "Offset: " - + std::to_string(offset) + ", Max: " + std::to_string(size_t(working_buffer.end() - working_buffer.begin())), + + std::to_string(offset) + ", Max: " + std::to_string(size_t(internal_buffer.end() - internal_buffer.begin())), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); } else if (whence == SEEK_CUR) { Position new_pos = pos + offset; - if (new_pos >= working_buffer.begin() && new_pos < working_buffer.end()) + if (new_pos >= internal_buffer.begin() && new_pos < internal_buffer.end()) { pos = new_pos; - return size_t(pos - working_buffer.begin()); + working_buffer = internal_buffer; /// We need to restore `working_buffer` in case the position was at EOF before this seek(). + return size_t(pos - internal_buffer.begin()); } else throw Exception( "Seek position is out of bounds. " "Offset: " - + std::to_string(offset) + ", Max: " + std::to_string(size_t(working_buffer.end() - working_buffer.begin())), + + std::to_string(offset) + ", Max: " + std::to_string(size_t(internal_buffer.end() - internal_buffer.begin())), ErrorCodes::SEEK_POSITION_OUT_OF_BOUND); } else @@ -45,7 +47,7 @@ off_t ReadBufferFromMemory::seek(off_t offset, int whence) off_t ReadBufferFromMemory::getPosition() { - return pos - working_buffer.begin(); + return pos - internal_buffer.begin(); } } diff --git a/src/IO/ReadBufferFromS3.cpp b/src/IO/ReadBufferFromS3.cpp index f01640cb95b..869432b9484 100644 --- a/src/IO/ReadBufferFromS3.cpp +++ b/src/IO/ReadBufferFromS3.cpp @@ -187,7 +187,7 @@ off_t ReadBufferFromS3::seek(off_t offset_, int whence) } } - pos = working_buffer.end(); + resetWorkingBuffer(); if (impl) { ProfileEvents::increment(ProfileEvents::ReadBufferSeekCancelConnection); diff --git a/src/IO/ReadHelpers.cpp b/src/IO/ReadHelpers.cpp index 4557f8a4251..a6125818155 100644 --- a/src/IO/ReadHelpers.cpp +++ b/src/IO/ReadHelpers.cpp @@ -216,6 +216,15 @@ void readStringUntilWhitespaceInto(Vector & s, ReadBuffer & buf) readStringUntilCharsInto<' '>(s, buf); } +template +void readStringUntilNewlineInto(Vector & s, ReadBuffer & buf) +{ + readStringUntilCharsInto<'\n'>(s, buf); +} + +template void readStringUntilNewlineInto>(PaddedPODArray & s, ReadBuffer & buf); +template void readStringUntilNewlineInto(String & s, ReadBuffer & buf); + template void readNullTerminated(Vector & s, ReadBuffer & buf) { diff --git a/src/IO/ReadHelpers.h b/src/IO/ReadHelpers.h index 73fe1a28be6..5d580f6b130 100644 --- a/src/IO/ReadHelpers.h +++ b/src/IO/ReadHelpers.h @@ -604,6 +604,9 @@ bool tryReadJSONStringInto(Vector & s, ReadBuffer & buf) template void readStringUntilWhitespaceInto(Vector & s, ReadBuffer & buf); +template +void readStringUntilNewlineInto(Vector & s, ReadBuffer & buf); + /// This could be used as template parameter for functions above, if you want to just skip data. struct NullOutput { @@ -1387,4 +1390,3 @@ void readQuotedFieldIntoString(String & s, ReadBuffer & buf); void readJSONFieldIntoString(String & s, ReadBuffer & buf); } - diff --git a/src/IO/ReadSettings.h b/src/IO/ReadSettings.h index f6c1158a896..e290cbab36b 100644 --- a/src/IO/ReadSettings.h +++ b/src/IO/ReadSettings.h @@ -82,6 +82,7 @@ struct ReadSettings size_t http_max_tries = 1; size_t http_retry_initial_backoff_ms = 100; size_t http_retry_max_backoff_ms = 1600; + bool http_skip_not_found_url_for_globs = true; /// Set to true for MergeTree tables to make sure /// that last position (offset in compressed file) is always passed. diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index 11c4e99c353..4e08a595484 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -129,6 +130,8 @@ namespace detail /// In case of redirects, save result uri to use it if we retry the request. std::optional saved_uri_redirect; + bool http_skip_not_found_url; + ReadSettings settings; Poco::Logger * log; @@ -146,7 +149,7 @@ namespace detail return read_range.begin + offset_from_begin_pos; } - std::istream * call(Poco::URI uri_, Poco::Net::HTTPResponse & response, const std::string & method_) + std::istream * callImpl(Poco::URI uri_, Poco::Net::HTTPResponse & response, const std::string & method_) { // With empty path poco will send "POST HTTP/1.1" its bug. if (uri_.getPath().empty()) @@ -211,7 +214,7 @@ namespace detail { try { - call(uri, response, Poco::Net::HTTPRequest::HTTP_HEAD); + call(response, Poco::Net::HTTPRequest::HTTP_HEAD); while (isRedirect(response.getStatus())) { @@ -220,7 +223,7 @@ namespace detail session->updateSession(uri_redirect); - istr = call(uri_redirect, response, method); + istr = callImpl(uri_redirect, response, method); } break; @@ -237,6 +240,17 @@ namespace detail return read_range.end; } + enum class InitializeError + { + /// If error is not retriable, `exception` variable must be set. + NON_RETRIABLE_ERROR, + /// Allows to skip not found urls for globs + SKIP_NOT_FOUND_URL, + NONE, + }; + + InitializeError initialization_error = InitializeError::NONE; + public: using NextCallback = std::function; using OutStreamCallback = std::function; @@ -253,7 +267,8 @@ namespace detail Range read_range_ = {}, const RemoteHostFilter & remote_host_filter_ = {}, bool delay_initialization = false, - bool use_external_buffer_ = false) + bool use_external_buffer_ = false, + bool http_skip_not_found_url_ = false) : SeekableReadBufferWithSize(nullptr, 0) , uri {uri_} , method {!method_.empty() ? method_ : out_stream_callback_ ? Poco::Net::HTTPRequest::HTTP_POST : Poco::Net::HTTPRequest::HTTP_GET} @@ -265,6 +280,7 @@ namespace detail , buffer_size {buffer_size_} , use_external_buffer {use_external_buffer_} , read_range(read_range_) + , http_skip_not_found_url(http_skip_not_found_url_) , settings {settings_} , log(&Poco::Logger::get("ReadWriteBufferFromHTTP")) { @@ -276,18 +292,58 @@ namespace detail "0 < http_retry_initial_backoff_ms < settings.http_retry_max_backoff_ms (now 0 < {} < {})", settings.http_max_tries, settings.http_retry_initial_backoff_ms, settings.http_retry_max_backoff_ms); + // Configure User-Agent if it not already set. + const std::string user_agent = "User-Agent"; + auto iter = std::find_if(http_header_entries.begin(), http_header_entries.end(), [&user_agent](const HTTPHeaderEntry & entry) + { + return std::get<0>(entry) == user_agent; + }); + + if (iter == http_header_entries.end()) + { + http_header_entries.emplace_back(std::make_pair("User-Agent", fmt::format("ClickHouse/{}", VERSION_STRING))); + } + if (!delay_initialization) + { initialize(); + if (exception) + std::rethrow_exception(exception); + } + } + + void call(Poco::Net::HTTPResponse & response, const String & method_) + { + try + { + istr = callImpl(saved_uri_redirect ? *saved_uri_redirect : uri, response, method_); + } + catch (...) + { + if (response.getStatus() == Poco::Net::HTTPResponse::HTTPStatus::HTTP_NOT_FOUND + && http_skip_not_found_url) + { + initialization_error = InitializeError::SKIP_NOT_FOUND_URL; + } + else + { + throw; + } + } } /** - * Note: In case of error return false if error is not retriable, otherwise throw. + * Throws if error is retriable, otherwise sets initialization_error = NON_RETRIABLE_ERROR and + * saves exception into `exception` variable. In case url is not found and skip_not_found_url == true, + * sets initialization_error = SKIP_NOT_FOUND_URL, otherwise throws. */ - bool initialize() + void initialize() { Poco::Net::HTTPResponse response; - istr = call(saved_uri_redirect ? *saved_uri_redirect : uri, response, method); + call(response, method); + if (initialization_error != InitializeError::NONE) + return; while (isRedirect(response.getStatus())) { @@ -296,7 +352,7 @@ namespace detail session->updateSession(uri_redirect); - istr = call(uri_redirect, response, method); + istr = callImpl(uri_redirect, response, method); saved_uri_redirect = uri_redirect; } @@ -310,7 +366,8 @@ namespace detail Exception(ErrorCodes::HTTP_RANGE_NOT_SATISFIABLE, "Cannot read with range: [{}, {}]", read_range.begin, read_range.end ? *read_range.end : '-')); - return false; + initialization_error = InitializeError::NON_RETRIABLE_ERROR; + return; } else if (read_range.end) { @@ -345,12 +402,14 @@ namespace detail sess->attachSessionData(e.message()); throw; } - - return true; } bool nextImpl() override { + if (initialization_error == InitializeError::SKIP_NOT_FOUND_URL) + return false; + assert(initialization_error == InitializeError::NONE); + if (next_callback) next_callback(count()); @@ -392,14 +451,16 @@ namespace detail { if (!impl) { - /// If error is not retriable -- false is returned and exception is set. - /// Otherwise the error is thrown and retries continue. - bool initialized = initialize(); - if (!initialized) + initialize(); + if (initialization_error == InitializeError::NON_RETRIABLE_ERROR) { assert(exception); break; } + else if (initialization_error == InitializeError::SKIP_NOT_FOUND_URL) + { + return false; + } if (use_external_buffer) { @@ -498,7 +559,7 @@ namespace detail impl.reset(); } - pos = working_buffer.end(); + resetWorkingBuffer(); read_range.begin = offset_; read_range.end = std::nullopt; offset_from_begin_pos = 0; @@ -537,7 +598,7 @@ class UpdatableSession : public UpdatableSessionBase using Parent = UpdatableSessionBase; public: - explicit UpdatableSession( + UpdatableSession( const Poco::URI uri, const ConnectionTimeouts & timeouts_, const UInt64 max_redirects_) @@ -557,7 +618,7 @@ class ReadWriteBufferFromHTTP : public detail::ReadWriteBufferFromHTTPBase>; public: - explicit ReadWriteBufferFromHTTP( + ReadWriteBufferFromHTTP( Poco::URI uri_, const std::string & method_, OutStreamCallback out_stream_callback_, @@ -570,11 +631,12 @@ public: Range read_range_ = {}, const RemoteHostFilter & remote_host_filter_ = {}, bool delay_initialization_ = true, - bool use_external_buffer_ = false) + bool use_external_buffer_ = false, + bool skip_not_found_url_ = false) : Parent(std::make_shared(uri_, timeouts, max_redirects), uri_, credentials_, method_, out_stream_callback_, buffer_size_, settings_, http_header_entries_, read_range_, remote_host_filter_, - delay_initialization_, use_external_buffer_) + delay_initialization_, use_external_buffer_, skip_not_found_url_) { } }; diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index 432dc443300..59a4dab837b 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -317,7 +317,7 @@ public: , load_frequency_ms(Aws::Auth::REFRESH_THRESHOLD) , logger(&Poco::Logger::get("AWSInstanceProfileCredentialsProvider")) { - LOG_INFO(logger, "Creating Instance with injected EC2MetadataClient and refresh rate {}."); + LOG_INFO(logger, "Creating Instance with injected EC2MetadataClient and refresh rate."); } Aws::Auth::AWSCredentials GetAWSCredentials() override diff --git a/src/IO/WriteBuffer.h b/src/IO/WriteBuffer.h index 9440ac0a855..bd2a628355e 100644 --- a/src/IO/WriteBuffer.h +++ b/src/IO/WriteBuffer.h @@ -104,10 +104,14 @@ public: ++pos; } - virtual void sync() - { - next(); - } + /// This method may be called before finalize() to tell there would not be any more data written. + /// Used does not have to call it, implementation should check it itself if needed. + /// + /// The idea is similar to prefetch. In case if all data is written, we can flush the buffer + /// and start sending data asynchronously. It may improve writing performance in case you have + /// multiple files to finalize. Mainly, for blob storage, finalization has high latency, + /// and calling preFinalize in a loop may parallelize it. + virtual void preFinalize() { next(); } /// Write the last data. void finalize() @@ -130,6 +134,13 @@ public: } } + /// Wait for data to be reliably written. Mainly, call fsync for fd. + /// May be called after finalize() if needed. + virtual void sync() + { + next(); + } + protected: virtual void finalizeImpl() { diff --git a/src/IO/WriteBufferFromFileDecorator.cpp b/src/IO/WriteBufferFromFileDecorator.cpp index 56d789ec23e..ac801534b4f 100644 --- a/src/IO/WriteBufferFromFileDecorator.cpp +++ b/src/IO/WriteBufferFromFileDecorator.cpp @@ -14,6 +14,10 @@ WriteBufferFromFileDecorator::WriteBufferFromFileDecorator(std::unique_ptrfinalize(); } diff --git a/src/IO/WriteBufferFromFileDecorator.h b/src/IO/WriteBufferFromFileDecorator.h index 18fdb8168f9..dde05276c28 100644 --- a/src/IO/WriteBufferFromFileDecorator.h +++ b/src/IO/WriteBufferFromFileDecorator.h @@ -17,6 +17,15 @@ public: std::string getFileName() const override; + void preFinalize() override + { + next(); + impl->preFinalize(); + is_prefinalized = true; + } + + const WriteBuffer & getImpl() const { return *impl; } + protected: void finalizeImpl() override; @@ -24,6 +33,8 @@ protected: private: void nextImpl() override; + + bool is_prefinalized = false; }; } diff --git a/src/IO/WriteBufferFromHTTP.cpp b/src/IO/WriteBufferFromHTTP.cpp index 5ddc28d2db1..622fab91fcc 100644 --- a/src/IO/WriteBufferFromHTTP.cpp +++ b/src/IO/WriteBufferFromHTTP.cpp @@ -10,6 +10,7 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( const Poco::URI & uri, const std::string & method, const std::string & content_type, + const std::string & content_encoding, const ConnectionTimeouts & timeouts, size_t buffer_size_) : WriteBufferFromOStream(buffer_size_) @@ -24,6 +25,9 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( request.set("Content-Type", content_type); } + if (!content_encoding.empty()) + request.set("Content-Encoding", content_encoding); + LOG_TRACE((&Poco::Logger::get("WriteBufferToHTTP")), "Sending request to {}", uri.toString()); ostr = &session->sendRequest(request); @@ -31,6 +35,10 @@ WriteBufferFromHTTP::WriteBufferFromHTTP( void WriteBufferFromHTTP::finalizeImpl() { + // for compressed body, the data is stored in buffered first + // here, make sure the content in the buffer has been flushed + this->nextImpl(); + receiveResponse(*session, request, response, false); /// TODO: Response body is ignored. } diff --git a/src/IO/WriteBufferFromHTTP.h b/src/IO/WriteBufferFromHTTP.h index 31b2a921889..6966bc8a5c5 100644 --- a/src/IO/WriteBufferFromHTTP.h +++ b/src/IO/WriteBufferFromHTTP.h @@ -21,6 +21,7 @@ public: explicit WriteBufferFromHTTP(const Poco::URI & uri, const std::string & method = Poco::Net::HTTPRequest::HTTP_POST, // POST or PUT only const std::string & content_type = "", + const std::string & content_encoding = "", const ConnectionTimeouts & timeouts = {}, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE); diff --git a/src/IO/WriteBufferFromS3.cpp b/src/IO/WriteBufferFromS3.cpp index 09193c4c396..d91b1e94297 100644 --- a/src/IO/WriteBufferFromS3.cpp +++ b/src/IO/WriteBufferFromS3.cpp @@ -34,22 +34,42 @@ namespace ErrorCodes extern const int S3_ERROR; } +struct WriteBufferFromS3::UploadPartTask +{ + Aws::S3::Model::UploadPartRequest req; + bool is_finised = false; + std::string tag; + std::exception_ptr exception; +}; + +struct WriteBufferFromS3::PutObjectTask +{ + Aws::S3::Model::PutObjectRequest req; + bool is_finised = false; + std::exception_ptr exception; +}; WriteBufferFromS3::WriteBufferFromS3( std::shared_ptr client_ptr_, const String & bucket_, const String & key_, size_t minimum_upload_part_size_, + size_t upload_part_size_multiply_factor_, + size_t upload_part_size_multiply_threshold_, size_t max_single_part_upload_size_, std::optional> object_metadata_, - size_t buffer_size_) + size_t buffer_size_, + ScheduleFunc schedule_) : BufferWithOwnMemory(buffer_size_, nullptr, 0) , bucket(bucket_) , key(key_) , object_metadata(std::move(object_metadata_)) , client_ptr(std::move(client_ptr_)) - , minimum_upload_part_size(minimum_upload_part_size_) + , upload_part_size(minimum_upload_part_size_) + , upload_part_size_multiply_factor(upload_part_size_multiply_factor_) + , upload_part_size_multiply_threshold(upload_part_size_multiply_threshold_) , max_single_part_upload_size(max_single_part_upload_size_) + , schedule(std::move(schedule_)) { allocateBuffer(); } @@ -59,6 +79,10 @@ void WriteBufferFromS3::nextImpl() if (!offset()) return; + /// Buffer in a bad state after exception + if (temporary_buffer->tellp() == -1) + allocateBuffer(); + temporary_buffer->write(working_buffer.begin(), offset()); ProfileEvents::increment(ProfileEvents::S3WriteBytes, offset()); @@ -69,15 +93,22 @@ void WriteBufferFromS3::nextImpl() if (multipart_upload_id.empty() && last_part_size > max_single_part_upload_size) createMultipartUpload(); - if (!multipart_upload_id.empty() && last_part_size > minimum_upload_part_size) + if (!multipart_upload_id.empty() && last_part_size > upload_part_size) { + writePart(); + allocateBuffer(); } + + waitForReadyBackGroundTasks(); } void WriteBufferFromS3::allocateBuffer() { + if (total_parts_uploaded != 0 && total_parts_uploaded % upload_part_size_multiply_threshold == 0) + upload_part_size *= upload_part_size_multiply_factor; + temporary_buffer = Aws::MakeShared("temporary buffer"); temporary_buffer->exceptions(std::ios::badbit); last_part_size = 0; @@ -88,7 +119,7 @@ WriteBufferFromS3::~WriteBufferFromS3() finalize(); } -void WriteBufferFromS3::finalizeImpl() +void WriteBufferFromS3::preFinalize() { next(); @@ -100,8 +131,20 @@ void WriteBufferFromS3::finalizeImpl() { /// Write rest of the data as last part. writePart(); - completeMultipartUpload(); } + + is_prefinalized = true; +} + +void WriteBufferFromS3::finalizeImpl() +{ + if (!is_prefinalized) + preFinalize(); + + waitForAllBackGroundTasks(); + + if (!multipart_upload_id.empty()) + completeMultipartUpload(); } void WriteBufferFromS3::createMultipartUpload() @@ -130,7 +173,10 @@ void WriteBufferFromS3::writePart() LOG_DEBUG(log, "Writing part. Bucket: {}, Key: {}, Upload_id: {}, Size: {}", bucket, key, multipart_upload_id, size); if (size < 0) - throw Exception("Failed to write part. Buffer in invalid state.", ErrorCodes::S3_ERROR); + { + LOG_WARNING(log, "Skipping part upload. Buffer is in bad state, it means that we have tried to upload something, but got an exception."); + return; + } if (size == 0) { @@ -144,25 +190,73 @@ void WriteBufferFromS3::writePart() LOG_WARNING(log, "Maximum part number in S3 protocol has reached (too many parts). Server may not accept this whole upload."); } - Aws::S3::Model::UploadPartRequest req; + if (schedule) + { + UploadPartTask * task = nullptr; + int part_number; + { + std::lock_guard lock(bg_tasks_mutex); + task = &upload_object_tasks.emplace_back(); + ++num_added_bg_tasks; + part_number = num_added_bg_tasks; + } + fillUploadRequest(task->req, part_number); + schedule([this, task]() + { + try + { + processUploadRequest(*task); + } + catch (...) + { + task->exception = std::current_exception(); + } + + { + std::lock_guard lock(bg_tasks_mutex); + task->is_finised = true; + ++num_finished_bg_tasks; + + /// Notification under mutex is important here. + /// Othervies, WriteBuffer could be destroyed in between + /// Releasing lock and condvar notification. + bg_tasks_condvar.notify_one(); + } + }); + } + else + { + UploadPartTask task; + fillUploadRequest(task.req, part_tags.size() + 1); + processUploadRequest(task); + part_tags.push_back(task.tag); + } +} + +void WriteBufferFromS3::fillUploadRequest(Aws::S3::Model::UploadPartRequest & req, int part_number) +{ req.SetBucket(bucket); req.SetKey(key); - req.SetPartNumber(part_tags.size() + 1); + req.SetPartNumber(part_number); req.SetUploadId(multipart_upload_id); - req.SetContentLength(size); + req.SetContentLength(temporary_buffer->tellp()); req.SetBody(temporary_buffer); +} - auto outcome = client_ptr->UploadPart(req); +void WriteBufferFromS3::processUploadRequest(UploadPartTask & task) +{ + auto outcome = client_ptr->UploadPart(task.req); if (outcome.IsSuccess()) { - auto etag = outcome.GetResult().GetETag(); - part_tags.push_back(etag); - LOG_DEBUG(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, etag, part_tags.size()); + task.tag = outcome.GetResult().GetETag(); + LOG_DEBUG(log, "Writing part finished. Bucket: {}, Key: {}, Upload_id: {}, Etag: {}, Parts: {}", bucket, key, multipart_upload_id, task.tag, part_tags.size()); } else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); + + total_parts_uploaded++; } void WriteBufferFromS3::completeMultipartUpload() @@ -191,17 +285,25 @@ void WriteBufferFromS3::completeMultipartUpload() if (outcome.IsSuccess()) LOG_DEBUG(log, "Multipart upload has completed. Bucket: {}, Key: {}, Upload_id: {}, Parts: {}", bucket, key, multipart_upload_id, part_tags.size()); else - throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); + { + throw Exception(ErrorCodes::S3_ERROR, "{} Tags:{}", + outcome.GetError().GetMessage(), + fmt::join(part_tags.begin(), part_tags.end(), " ")); + } } void WriteBufferFromS3::makeSinglepartUpload() { auto size = temporary_buffer->tellp(); + bool with_pool = bool(schedule); - LOG_DEBUG(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}", bucket, key, size); + LOG_DEBUG(log, "Making single part upload. Bucket: {}, Key: {}, Size: {}, WithPool: {}", bucket, key, size, with_pool); if (size < 0) - throw Exception("Failed to make single part upload. Buffer in invalid state", ErrorCodes::S3_ERROR); + { + LOG_WARNING(log, "Skipping single part upload. Buffer is in bad state, it mean that we have tried to upload something, but got an exception."); + return; + } if (size == 0) { @@ -209,22 +311,114 @@ void WriteBufferFromS3::makeSinglepartUpload() return; } - Aws::S3::Model::PutObjectRequest req; + if (schedule) + { + put_object_task = std::make_unique(); + fillPutRequest(put_object_task->req); + schedule([this]() + { + try + { + processPutRequest(*put_object_task); + } + catch (...) + { + put_object_task->exception = std::current_exception(); + } + + { + std::lock_guard lock(bg_tasks_mutex); + put_object_task->is_finised = true; + + /// Notification under mutex is important here. + /// Othervies, WriteBuffer could be destroyed in between + /// Releasing lock and condvar notification. + bg_tasks_condvar.notify_one(); + } + + }); + } + else + { + PutObjectTask task; + fillPutRequest(task.req); + processPutRequest(task); + } +} + +void WriteBufferFromS3::fillPutRequest(Aws::S3::Model::PutObjectRequest & req) +{ req.SetBucket(bucket); req.SetKey(key); - req.SetContentLength(size); + req.SetContentLength(temporary_buffer->tellp()); req.SetBody(temporary_buffer); if (object_metadata.has_value()) req.SetMetadata(object_metadata.value()); +} - auto outcome = client_ptr->PutObject(req); +void WriteBufferFromS3::processPutRequest(PutObjectTask & task) +{ + auto outcome = client_ptr->PutObject(task.req); + bool with_pool = bool(schedule); if (outcome.IsSuccess()) - LOG_DEBUG(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}", bucket, key, req.GetContentLength()); + LOG_DEBUG(log, "Single part upload has completed. Bucket: {}, Key: {}, Object size: {}, WithPool: {}", bucket, key, task.req.GetContentLength(), with_pool); else throw Exception(outcome.GetError().GetMessage(), ErrorCodes::S3_ERROR); } +void WriteBufferFromS3::waitForReadyBackGroundTasks() +{ + if (schedule) + { + std::lock_guard lock(bg_tasks_mutex); + { + while (!upload_object_tasks.empty() && upload_object_tasks.front().is_finised) + { + auto & task = upload_object_tasks.front(); + auto exception = std::move(task.exception); + auto tag = std::move(task.tag); + upload_object_tasks.pop_front(); + + if (exception) + { + waitForAllBackGroundTasks(); + std::rethrow_exception(exception); + } + + part_tags.push_back(tag); + } + } + } +} + +void WriteBufferFromS3::waitForAllBackGroundTasks() +{ + if (schedule) + { + std::unique_lock lock(bg_tasks_mutex); + bg_tasks_condvar.wait(lock, [this]() { return num_added_bg_tasks == num_finished_bg_tasks; }); + + while (!upload_object_tasks.empty()) + { + auto & task = upload_object_tasks.front(); + if (task.exception) + std::rethrow_exception(std::move(task.exception)); + + part_tags.push_back(task.tag); + + upload_object_tasks.pop_front(); + } + + if (put_object_task) + { + bg_tasks_condvar.wait(lock, [this]() { return put_object_task->is_finised; }); + if (put_object_task->exception) + std::rethrow_exception(std::move(put_object_task->exception)); + } + } +} + } #endif diff --git a/src/IO/WriteBufferFromS3.h b/src/IO/WriteBufferFromS3.h index e4633282061..8b89626ee18 100644 --- a/src/IO/WriteBufferFromS3.h +++ b/src/IO/WriteBufferFromS3.h @@ -14,14 +14,24 @@ # include +# include + namespace Aws::S3 { class S3Client; } +namespace Aws::S3::Model +{ + class UploadPartRequest; + class PutObjectRequest; +} + namespace DB { +using ScheduleFunc = std::function)>; + /** * Buffer to write a data to a S3 object with specified bucket and key. * If data size written to the buffer is less than 'max_single_part_upload_size' write is performed using singlepart upload. @@ -29,7 +39,7 @@ namespace DB * Data is divided on chunks with size greater than 'minimum_upload_part_size'. Last chunk can be less than this threshold. * Each chunk is written as a part to S3. */ -class WriteBufferFromS3 : public BufferWithOwnMemory +class WriteBufferFromS3 final : public BufferWithOwnMemory { public: explicit WriteBufferFromS3( @@ -37,14 +47,19 @@ public: const String & bucket_, const String & key_, size_t minimum_upload_part_size_, + size_t upload_part_size_multiply_factor_, + size_t upload_part_size_multiply_threshold_, size_t max_single_part_upload_size_, std::optional> object_metadata_ = std::nullopt, - size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE); + size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, + ScheduleFunc schedule_ = {}); ~WriteBufferFromS3() override; void nextImpl() override; + void preFinalize() override; + private: void allocateBuffer(); @@ -57,21 +72,47 @@ private: /// Receives response from the server after sending all data. void finalizeImpl() override; + struct UploadPartTask; + void fillUploadRequest(Aws::S3::Model::UploadPartRequest & req, int part_number); + void processUploadRequest(UploadPartTask & task); + + struct PutObjectTask; + void fillPutRequest(Aws::S3::Model::PutObjectRequest & req); + void processPutRequest(PutObjectTask & task); + + void waitForReadyBackGroundTasks(); + void waitForAllBackGroundTasks(); + String bucket; String key; std::optional> object_metadata; std::shared_ptr client_ptr; - size_t minimum_upload_part_size; - size_t max_single_part_upload_size; + size_t upload_part_size; + const size_t upload_part_size_multiply_factor; + const size_t upload_part_size_multiply_threshold; + const size_t max_single_part_upload_size; /// Buffer to accumulate data. std::shared_ptr temporary_buffer; - size_t last_part_size; + size_t last_part_size = 0; + std::atomic total_parts_uploaded = 0; /// Upload in S3 is made in parts. /// We initiate upload, then upload each part and get ETag as a response, and then finalizeImpl() upload with listing all our parts. String multipart_upload_id; std::vector part_tags; + bool is_prefinalized = false; + + /// Following fields are for background uploads in thread pool (if specified). + /// We use std::function to avoid dependency of Interpreters + ScheduleFunc schedule; + std::unique_ptr put_object_task; + std::list upload_object_tasks; + size_t num_added_bg_tasks = 0; + size_t num_finished_bg_tasks = 0; + std::mutex bg_tasks_mutex; + std::condition_variable bg_tasks_condvar; + Poco::Logger * log = &Poco::Logger::get("WriteBufferFromS3"); }; diff --git a/src/IO/parseDateTimeBestEffort.cpp b/src/IO/parseDateTimeBestEffort.cpp index 3b05d8c76b6..3c6f9b8f9f5 100644 --- a/src/IO/parseDateTimeBestEffort.cpp +++ b/src/IO/parseDateTimeBestEffort.cpp @@ -194,7 +194,7 @@ ReturnType parseDateTimeBestEffortImpl( } else if (num_digits == 6) { - /// This is YYYYMM + /// This is YYYYMM or hhmmss if (!year && !month) { readDecimalNumber<4>(year, digits); @@ -435,47 +435,59 @@ ReturnType parseDateTimeBestEffortImpl( else if (c == '+' || c == '-') { ++in.position(); - has_time_zone_offset = true; - if (c == '-') - time_zone_offset_negative = true; - num_digits = readDigits(digits, sizeof(digits), in); - if (num_digits == 4) + if (num_digits == 6 && !has_time && year && month && day_of_month) { - readDecimalNumber<2>(time_zone_offset_hour, digits); - readDecimalNumber<2>(time_zone_offset_minute, digits + 2); - } - else if (num_digits == 3) - { - readDecimalNumber<1>(time_zone_offset_hour, digits); - readDecimalNumber<2>(time_zone_offset_minute, digits + 1); - } - else if (num_digits == 2) - { - readDecimalNumber<2>(time_zone_offset_hour, digits); - } - else if (num_digits == 1) - { - readDecimalNumber<1>(time_zone_offset_hour, digits); + /// It looks like hhmmss + readDecimalNumber<2>(hour, digits); + readDecimalNumber<2>(minute, digits + 2); + readDecimalNumber<2>(second, digits + 4); + has_time = true; } else - return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME); - - if (num_digits < 3 && checkChar(':', in)) { - num_digits = readDigits(digits, sizeof(digits), in); + /// It looks like time zone offset + has_time_zone_offset = true; + if (c == '-') + time_zone_offset_negative = true; - if (num_digits == 2) + if (num_digits == 4) { - readDecimalNumber<2>(time_zone_offset_minute, digits); + readDecimalNumber<2>(time_zone_offset_hour, digits); + readDecimalNumber<2>(time_zone_offset_minute, digits + 2); + } + else if (num_digits == 3) + { + readDecimalNumber<1>(time_zone_offset_hour, digits); + readDecimalNumber<2>(time_zone_offset_minute, digits + 1); + } + else if (num_digits == 2) + { + readDecimalNumber<2>(time_zone_offset_hour, digits); } else if (num_digits == 1) { - readDecimalNumber<1>(time_zone_offset_minute, digits); + readDecimalNumber<1>(time_zone_offset_hour, digits); } else - return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset in minutes: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME); + return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME); + + if (num_digits < 3 && checkChar(':', in)) + { + num_digits = readDigits(digits, sizeof(digits), in); + + if (num_digits == 2) + { + readDecimalNumber<2>(time_zone_offset_minute, digits); + } + else if (num_digits == 1) + { + readDecimalNumber<1>(time_zone_offset_minute, digits); + } + else + return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset in minutes: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME); + } } } else diff --git a/src/IO/tests/gtest_archive_reader_and_writer.cpp b/src/IO/tests/gtest_archive_reader_and_writer.cpp new file mode 100644 index 00000000000..e1864415e1b --- /dev/null +++ b/src/IO/tests/gtest_archive_reader_and_writer.cpp @@ -0,0 +1,343 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + + +namespace DB::ErrorCodes +{ + extern const int CANNOT_UNPACK_ARCHIVE; +} + +namespace fs = std::filesystem; +using namespace DB; + + +class ArchiveReaderAndWriterTest : public ::testing::TestWithParam +{ +public: + ArchiveReaderAndWriterTest() + { + const char * archive_file_ext = GetParam(); + path_to_archive = temp_folder.path() + "/archive" + archive_file_ext; + fs::create_directories(temp_folder.path()); + } + + const String & getPathToArchive() const { return path_to_archive; } + + static void expectException(int code, const String & message, const std::function & func) + { + try + { + func(); + } + catch (Exception & e) + { + if ((e.code() != code) || (e.message().find(message) == String::npos)) + throw; + } + } + +private: + Poco::TemporaryFile temp_folder; + String path_to_archive; +}; + + +TEST_P(ArchiveReaderAndWriterTest, EmptyArchive) +{ + /// Make an archive. + { + createArchiveWriter(getPathToArchive()); + } + + /// The created archive can be found in the local filesystem. + ASSERT_TRUE(fs::exists(getPathToArchive())); + + /// Read the archive. + auto reader = createArchiveReader(getPathToArchive()); + + EXPECT_FALSE(reader->fileExists("nofile.txt")); + + expectException(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "File 'nofile.txt' not found", + [&]{ reader->getFileInfo("nofile.txt"); }); + + expectException(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "File 'nofile.txt' not found", + [&]{ reader->readFile("nofile.txt"); }); + + EXPECT_EQ(reader->firstFile(), nullptr); +} + + +TEST_P(ArchiveReaderAndWriterTest, SingleFileInArchive) +{ + /// Make an archive. + std::string_view contents = "The contents of a.txt"; + { + auto writer = createArchiveWriter(getPathToArchive()); + { + auto out = writer->writeFile("a.txt"); + writeString(contents, *out); + } + } + + /// Read the archive. + auto reader = createArchiveReader(getPathToArchive()); + + ASSERT_TRUE(reader->fileExists("a.txt")); + + auto file_info = reader->getFileInfo("a.txt"); + EXPECT_EQ(file_info.uncompressed_size, contents.size()); + EXPECT_GT(file_info.compressed_size, 0); + + { + auto in = reader->readFile("a.txt"); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, contents); + } + + { + /// Use an enumerator. + auto enumerator = reader->firstFile(); + ASSERT_NE(enumerator, nullptr); + EXPECT_EQ(enumerator->getFileName(), "a.txt"); + EXPECT_EQ(enumerator->getFileInfo().uncompressed_size, contents.size()); + EXPECT_GT(enumerator->getFileInfo().compressed_size, 0); + EXPECT_FALSE(enumerator->nextFile()); + } + + { + /// Use converting an enumerator to a reading buffer and vice versa. + auto enumerator = reader->firstFile(); + ASSERT_NE(enumerator, nullptr); + EXPECT_EQ(enumerator->getFileName(), "a.txt"); + auto in = reader->readFile(std::move(enumerator)); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, contents); + enumerator = reader->nextFile(std::move(in)); + EXPECT_EQ(enumerator, nullptr); + } + + { + /// Wrong using of an enumerator throws an exception. + auto enumerator = reader->firstFile(); + ASSERT_NE(enumerator, nullptr); + EXPECT_FALSE(enumerator->nextFile()); + expectException(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "No current file", + [&]{ enumerator->getFileName(); }); + + expectException(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "No current file", + [&] { reader->readFile(std::move(enumerator)); }); + } +} + + +TEST_P(ArchiveReaderAndWriterTest, TwoFilesInArchive) +{ + /// Make an archive. + std::string_view a_contents = "The contents of a.txt"; + std::string_view c_contents = "The contents of b/c.txt"; + { + auto writer = createArchiveWriter(getPathToArchive()); + { + auto out = writer->writeFile("a.txt"); + writeString(a_contents, *out); + } + { + auto out = writer->writeFile("b/c.txt"); + writeString(c_contents, *out); + } + } + + /// Read the archive. + auto reader = createArchiveReader(getPathToArchive()); + + ASSERT_TRUE(reader->fileExists("a.txt")); + ASSERT_TRUE(reader->fileExists("b/c.txt")); + + EXPECT_EQ(reader->getFileInfo("a.txt").uncompressed_size, a_contents.size()); + EXPECT_EQ(reader->getFileInfo("b/c.txt").uncompressed_size, c_contents.size()); + + { + auto in = reader->readFile("a.txt"); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, a_contents); + } + + { + auto in = reader->readFile("b/c.txt"); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, c_contents); + } + + { + /// Read a.txt again. + auto in = reader->readFile("a.txt"); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, a_contents); + } + + { + /// Use an enumerator. + auto enumerator = reader->firstFile(); + ASSERT_NE(enumerator, nullptr); + EXPECT_EQ(enumerator->getFileName(), "a.txt"); + EXPECT_EQ(enumerator->getFileInfo().uncompressed_size, a_contents.size()); + EXPECT_TRUE(enumerator->nextFile()); + EXPECT_EQ(enumerator->getFileName(), "b/c.txt"); + EXPECT_EQ(enumerator->getFileInfo().uncompressed_size, c_contents.size()); + EXPECT_FALSE(enumerator->nextFile()); + } + + { + /// Use converting an enumerator to a reading buffer and vice versa. + auto enumerator = reader->firstFile(); + ASSERT_NE(enumerator, nullptr); + EXPECT_EQ(enumerator->getFileName(), "a.txt"); + auto in = reader->readFile(std::move(enumerator)); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, a_contents); + enumerator = reader->nextFile(std::move(in)); + ASSERT_NE(enumerator, nullptr); + EXPECT_EQ(enumerator->getFileName(), "b/c.txt"); + in = reader->readFile(std::move(enumerator)); + readStringUntilEOF(str, *in); + EXPECT_EQ(str, c_contents); + enumerator = reader->nextFile(std::move(in)); + EXPECT_EQ(enumerator, nullptr); + } +} + + +TEST_P(ArchiveReaderAndWriterTest, InMemory) +{ + String archive_in_memory; + + /// Make an archive. + std::string_view a_contents = "The contents of a.txt"; + std::string_view b_contents = "The contents of b.txt"; + { + auto writer = createArchiveWriter(getPathToArchive(), std::make_unique(archive_in_memory)); + { + auto out = writer->writeFile("a.txt"); + writeString(a_contents, *out); + } + { + auto out = writer->writeFile("b.txt"); + writeString(b_contents, *out); + } + } + + /// The created archive is really in memory. + ASSERT_FALSE(fs::exists(getPathToArchive())); + + /// Read the archive. + auto read_archive_func = [&]() -> std::unique_ptr { return std::make_unique(archive_in_memory); }; + auto reader = createArchiveReader(getPathToArchive(), read_archive_func, archive_in_memory.size()); + + ASSERT_TRUE(reader->fileExists("a.txt")); + ASSERT_TRUE(reader->fileExists("b.txt")); + + EXPECT_EQ(reader->getFileInfo("a.txt").uncompressed_size, a_contents.size()); + EXPECT_EQ(reader->getFileInfo("b.txt").uncompressed_size, b_contents.size()); + + { + auto in = reader->readFile("a.txt"); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, a_contents); + } + + { + auto in = reader->readFile("b.txt"); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, b_contents); + } + + { + /// Read a.txt again. + auto in = reader->readFile("a.txt"); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, a_contents); + } +} + + +TEST_P(ArchiveReaderAndWriterTest, Password) +{ + /// Make an archive. + std::string_view contents = "The contents of a.txt"; + { + auto writer = createArchiveWriter(getPathToArchive()); + writer->setPassword("Qwe123"); + { + auto out = writer->writeFile("a.txt"); + writeString(contents, *out); + } + } + + /// Read the archive. + auto reader = createArchiveReader(getPathToArchive()); + + /// Try to read without a password. + expectException(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "Password is required", + [&]{ reader->readFile("a.txt"); }); + + { + /// Try to read with a wrong password. + reader->setPassword("123Qwe"); + expectException(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "Wrong password", + [&]{ reader->readFile("a.txt"); }); + } + + { + /// Reading with the right password is successful. + reader->setPassword("Qwe123"); + auto in = reader->readFile("a.txt"); + String str; + readStringUntilEOF(str, *in); + EXPECT_EQ(str, contents); + } +} + + +TEST_P(ArchiveReaderAndWriterTest, ArchiveNotExist) +{ + expectException(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "Couldn't open", + [&]{ createArchiveReader(getPathToArchive()); }); +} + + +#if USE_MINIZIP + +namespace +{ + const char * supported_archive_file_exts[] = + { + ".zip", + }; +} + +INSTANTIATE_TEST_SUITE_P(All, ArchiveReaderAndWriterTest, ::testing::ValuesIn(supported_archive_file_exts)); + +#endif diff --git a/src/IO/tests/gtest_seek_backwards.cpp b/src/IO/tests/gtest_seek_backwards.cpp new file mode 100644 index 00000000000..93aab68936f --- /dev/null +++ b/src/IO/tests/gtest_seek_backwards.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include +#include +#include + +using namespace DB; + +TEST(ReadBufferFromFile, seekBackwards) +{ + static constexpr size_t N = 256; + static constexpr size_t BUF_SIZE = 64; + + auto tmp_file = createTemporaryFile("/tmp/"); + + { + WriteBufferFromFile out(tmp_file->path()); + for (size_t i = 0; i < N; ++i) + writeIntBinary(i, out); + } + + ReadBufferFromFile in(tmp_file->path(), BUF_SIZE); + size_t x; + + /// Read something to initialize the buffer. + in.seek(BUF_SIZE * 10, SEEK_SET); + readIntBinary(x, in); + + /// Check 2 consecutive seek calls without reading. + in.seek(BUF_SIZE * 2, SEEK_SET); + in.seek(BUF_SIZE, SEEK_SET); + + readIntBinary(x, in); + ASSERT_EQ(x, 8); +} diff --git a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp index c88e9c299a8..72b4b149bd7 100644 --- a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp +++ b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -45,22 +46,24 @@ namespace BlockIO InterpreterCreateRowPolicyQuery::execute() { auto & query = query_ptr->as(); - auto & access_control = getContext()->getAccessControl(); - getContext()->checkAccess(query.alter ? AccessType::ALTER_ROW_POLICY : AccessType::CREATE_ROW_POLICY); + auto required_access = getRequiredAccess(); if (!query.cluster.empty()) { query.replaceCurrentUserTag(getContext()->getUserName()); - return executeDDLQueryOnCluster(query_ptr, getContext()); + return executeDDLQueryOnCluster(query_ptr, getContext(), required_access); } assert(query.names->cluster.empty()); + auto & access_control = getContext()->getAccessControl(); + getContext()->checkAccess(required_access); + + query.replaceEmptyDatabase(getContext()->getCurrentDatabase()); + std::optional roles_from_query; if (query.roles) roles_from_query = RolesOrUsersSet{*query.roles, access_control, getContext()->getUserID()}; - query.replaceEmptyDatabase(getContext()->getCurrentDatabase()); - if (query.alter) { auto update_func = [&](const AccessEntityPtr & entity) -> AccessEntityPtr @@ -105,4 +108,15 @@ void InterpreterCreateRowPolicyQuery::updateRowPolicyFromQuery(RowPolicy & polic updateRowPolicyFromQueryImpl(policy, query, {}, {}); } + +AccessRightsElements InterpreterCreateRowPolicyQuery::getRequiredAccess() const +{ + const auto & query = query_ptr->as(); + AccessRightsElements res; + auto access_type = (query.alter ? AccessType::ALTER_ROW_POLICY : AccessType::CREATE_ROW_POLICY); + for (const auto & row_policy_name : query.names->full_names) + res.emplace_back(access_type, row_policy_name.database, row_policy_name.table_name); + return res; +} + } diff --git a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h index 8adfe6b0855..e76cc1c165d 100644 --- a/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h +++ b/src/Interpreters/Access/InterpreterCreateRowPolicyQuery.h @@ -6,8 +6,8 @@ namespace DB { - class ASTCreateRowPolicyQuery; +class AccessRightsElements; struct RowPolicy; class InterpreterCreateRowPolicyQuery : public IInterpreter, WithMutableContext @@ -20,6 +20,8 @@ public: static void updateRowPolicyFromQuery(RowPolicy & policy, const ASTCreateRowPolicyQuery & query); private: + AccessRightsElements getRequiredAccess() const; + ASTPtr query_ptr; }; diff --git a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp index 4d2e880561e..3437e7fe0f4 100644 --- a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.cpp @@ -49,12 +49,37 @@ AccessRightsElements InterpreterDropAccessEntityQuery::getRequiredAccess() const AccessRightsElements res; switch (query.type) { - case AccessEntityType::USER: res.emplace_back(AccessType::DROP_USER); return res; - case AccessEntityType::ROLE: res.emplace_back(AccessType::DROP_ROLE); return res; - case AccessEntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::DROP_SETTINGS_PROFILE); return res; - case AccessEntityType::ROW_POLICY: res.emplace_back(AccessType::DROP_ROW_POLICY); return res; - case AccessEntityType::QUOTA: res.emplace_back(AccessType::DROP_QUOTA); return res; - case AccessEntityType::MAX: break; + case AccessEntityType::USER: + { + res.emplace_back(AccessType::DROP_USER); + return res; + } + case AccessEntityType::ROLE: + { + res.emplace_back(AccessType::DROP_ROLE); + return res; + } + case AccessEntityType::SETTINGS_PROFILE: + { + res.emplace_back(AccessType::DROP_SETTINGS_PROFILE); + return res; + } + case AccessEntityType::ROW_POLICY: + { + if (query.row_policy_names) + { + for (const auto & row_policy_name : query.row_policy_names->full_names) + res.emplace_back(AccessType::DROP_ROW_POLICY, row_policy_name.database, row_policy_name.table_name); + } + return res; + } + case AccessEntityType::QUOTA: + { + res.emplace_back(AccessType::DROP_QUOTA); + return res; + } + case AccessEntityType::MAX: + break; } throw Exception( toString(query.type) + ": type is not supported by DROP query", ErrorCodes::NOT_IMPLEMENTED); diff --git a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.h b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.h index 0ee478e904e..ea2d127913f 100644 --- a/src/Interpreters/Access/InterpreterDropAccessEntityQuery.h +++ b/src/Interpreters/Access/InterpreterDropAccessEntityQuery.h @@ -6,7 +6,6 @@ namespace DB { - class AccessRightsElements; class InterpreterDropAccessEntityQuery : public IInterpreter, WithMutableContext diff --git a/src/Interpreters/Access/InterpreterShowAccessQuery.cpp b/src/Interpreters/Access/InterpreterShowAccessQuery.cpp index 26c47507ce2..e16ee03c711 100644 --- a/src/Interpreters/Access/InterpreterShowAccessQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowAccessQuery.cpp @@ -10,7 +10,7 @@ #include #include #include -#include +#include #include @@ -61,7 +61,7 @@ std::vector InterpreterShowAccessQuery::getEntities() const } } - boost::range::sort(entities, IAccessEntity::LessByTypeAndName{}); + ::sort(entities.begin(), entities.end(), IAccessEntity::LessByTypeAndName{}); return entities; } diff --git a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp index 284b3cd1b48..27345218e07 100644 --- a/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowCreateAccessEntityQuery.cpp @@ -27,7 +27,7 @@ #include #include #include -#include +#include namespace DB @@ -341,7 +341,7 @@ std::vector InterpreterShowCreateAccessEntityQuery::getEntities entities.push_back(access_control.read(access_control.getID(show_query.type, name))); } - boost::range::sort(entities, IAccessEntity::LessByName{}); + ::sort(entities.begin(), entities.end(), IAccessEntity::LessByName{}); return entities; } @@ -377,12 +377,48 @@ AccessRightsElements InterpreterShowCreateAccessEntityQuery::getRequiredAccess() AccessRightsElements res; switch (show_query.type) { - case AccessEntityType::USER: res.emplace_back(AccessType::SHOW_USERS); return res; - case AccessEntityType::ROLE: res.emplace_back(AccessType::SHOW_ROLES); return res; - case AccessEntityType::SETTINGS_PROFILE: res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); return res; - case AccessEntityType::ROW_POLICY: res.emplace_back(AccessType::SHOW_ROW_POLICIES); return res; - case AccessEntityType::QUOTA: res.emplace_back(AccessType::SHOW_QUOTAS); return res; - case AccessEntityType::MAX: break; + case AccessEntityType::USER: + { + res.emplace_back(AccessType::SHOW_USERS); + return res; + } + case AccessEntityType::ROLE: + { + res.emplace_back(AccessType::SHOW_ROLES); + return res; + } + case AccessEntityType::SETTINGS_PROFILE: + { + res.emplace_back(AccessType::SHOW_SETTINGS_PROFILES); + return res; + } + case AccessEntityType::ROW_POLICY: + { + if (show_query.row_policy_names) + { + for (const auto & row_policy_name : show_query.row_policy_names->full_names) + res.emplace_back(AccessType::SHOW_ROW_POLICIES, row_policy_name.database, row_policy_name.table_name); + } + else if (show_query.database_and_table_name) + { + if (show_query.database_and_table_name->second.empty()) + res.emplace_back(AccessType::SHOW_ROW_POLICIES, show_query.database_and_table_name->first); + else + res.emplace_back(AccessType::SHOW_ROW_POLICIES, show_query.database_and_table_name->first, show_query.database_and_table_name->second); + } + else + { + res.emplace_back(AccessType::SHOW_ROW_POLICIES); + } + return res; + } + case AccessEntityType::QUOTA: + { + res.emplace_back(AccessType::SHOW_QUOTAS); + return res; + } + case AccessEntityType::MAX: + break; } throw Exception(toString(show_query.type) + ": type is not supported by SHOW CREATE query", ErrorCodes::NOT_IMPLEMENTED); } diff --git a/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp b/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp index cd98d8d4575..c82088847d3 100644 --- a/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp +++ b/src/Interpreters/Access/InterpreterShowGrantsQuery.cpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include #include @@ -11,8 +13,8 @@ #include #include #include -#include #include +#include namespace DB @@ -135,19 +137,29 @@ QueryPipeline InterpreterShowGrantsQuery::executeImpl() std::vector InterpreterShowGrantsQuery::getEntities() const { - const auto & show_query = query_ptr->as(); + const auto & access = getContext()->getAccess(); const auto & access_control = getContext()->getAccessControl(); + + const auto & show_query = query_ptr->as(); auto ids = RolesOrUsersSet{*show_query.for_roles, access_control, getContext()->getUserID()}.getMatchingIDs(access_control); + CachedAccessChecking show_users(access, AccessType::SHOW_USERS); + CachedAccessChecking show_roles(access, AccessType::SHOW_ROLES); + bool throw_if_access_denied = !show_query.for_roles->all; + std::vector entities; for (const auto & id : ids) { auto entity = access_control.tryRead(id); - if (entity) + if (!entity) + continue; + if ((id == access->getUserID() /* Any user can see his own grants */) + || (entity->isTypeOf() && show_users.checkAccess(throw_if_access_denied)) + || (entity->isTypeOf() && show_roles.checkAccess(throw_if_access_denied))) entities.push_back(entity); } - boost::range::sort(entities, IAccessEntity::LessByTypeAndName{}); + ::sort(entities.begin(), entities.end(), IAccessEntity::LessByTypeAndName{}); return entities; } diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index d0b360dda82..6ed35210251 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -13,6 +13,7 @@ #include #include +#include #include namespace DB @@ -601,8 +602,8 @@ NameSet ActionsDAG::foldActionsByProjection( std::unordered_set visited_nodes; std::unordered_set visited_index_names; std::stack stack; - std::vector missing_input_from_projection_keys; + /// Record all needed index nodes to start folding. for (const auto & node : index) { if (required_columns.find(node->result_name) != required_columns.end() || node->result_name == predicate_column_name) @@ -613,6 +614,9 @@ NameSet ActionsDAG::foldActionsByProjection( } } + /// If some required columns are not in any index node, try searching from all projection key + /// columns. If still missing, return empty set which means current projection fails to match + /// (missing columns). if (add_missing_keys) { for (const auto & column : required_columns) @@ -635,6 +639,7 @@ NameSet ActionsDAG::foldActionsByProjection( } } + /// Traverse the DAG from root to leaf. Substitute any matched node with columns in projection_block_for_keys. while (!stack.empty()) { auto * node = stack.top(); @@ -663,10 +668,12 @@ NameSet ActionsDAG::foldActionsByProjection( } } + /// Clean up unused nodes after folding. std::erase_if(inputs, [&](const Node * node) { return visited_nodes.count(node) == 0; }); std::erase_if(index, [&](const Node * node) { return visited_index_names.count(node->result_name) == 0; }); nodes.remove_if([&](const Node & node) { return visited_nodes.count(&node) == 0; }); + /// Calculate the required columns after folding. NameSet next_required_columns; for (const auto & input : inputs) next_required_columns.insert(input->result_name); @@ -676,7 +683,7 @@ NameSet ActionsDAG::foldActionsByProjection( void ActionsDAG::reorderAggregationKeysForProjection(const std::unordered_map & key_names_pos_map) { - std::sort(index.begin(), index.end(), [&key_names_pos_map](const Node * lhs, const Node * rhs) + ::sort(index.begin(), index.end(), [&key_names_pos_map](const Node * lhs, const Node * rhs) { return key_names_pos_map.find(lhs->result_name)->second < key_names_pos_map.find(rhs->result_name)->second; }); diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 9a5ad01a252..b07ab08c997 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -163,12 +163,43 @@ public: void removeUnusedActions(const Names & required_names, bool allow_remove_inputs = true, bool allow_constant_folding = true); void removeUnusedActions(const NameSet & required_names, bool allow_remove_inputs = true, bool allow_constant_folding = true); + /// Transform the current DAG in a way that leaf nodes get folded into their parents. It's done + /// because each projection can provide some columns as inputs to substitute certain sub-DAGs + /// (expressions). Consider the following example: + /// CREATE TABLE tbl (dt DateTime, val UInt64, + /// PROJECTION p_hour (SELECT SUM(val) GROUP BY toStartOfHour(dt))); + /// + /// Query: SELECT toStartOfHour(dt), SUM(val) FROM tbl GROUP BY toStartOfHour(dt); + /// + /// We will have an ActionsDAG like this: + /// FUNCTION: toStartOfHour(dt) SUM(val) + /// ^ ^ + /// | | + /// INPUT: dt val + /// + /// Now we traverse the DAG and see if any FUNCTION node can be replaced by projection's INPUT node. + /// The result DAG will be: + /// INPUT: toStartOfHour(dt) SUM(val) + /// + /// We don't need aggregate columns from projection because they are matched after DAG. + /// Currently we use canonical names of each node to find matches. It can be improved after we + /// have a full-featured name binding system. + /// + /// @param required_columns should contain columns which this DAG is required to produce after folding. It used for result actions. + /// @param projection_block_for_keys contains all key columns of given projection. + /// @param predicate_column_name means we need to produce the predicate column after folding. + /// @param add_missing_keys means whether to add additional missing columns to input nodes from projection key columns directly. + /// @return required columns for this folded DAG. It's expected to be fewer than the original ones if some projection is used. NameSet foldActionsByProjection( const NameSet & required_columns, const Block & projection_block_for_keys, const String & predicate_column_name = {}, bool add_missing_keys = true); + + /// Reorder the index nodes using given position mapping. void reorderAggregationKeysForProjection(const std::unordered_map & key_names_pos_map); + + /// Add aggregate columns to index nodes from projection void addAggregatesViaProjection(const Block & aggregates); bool hasArrayJoin() const; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 5c9d94d7c45..a2f24a79e40 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -854,12 +855,18 @@ void NO_INLINE Aggregator::executeWithoutKeyImpl( void NO_INLINE Aggregator::executeOnIntervalWithoutKeyImpl( - AggregatedDataWithoutKey & res, + AggregatedDataVariants & data_variants, size_t row_begin, size_t row_end, AggregateFunctionInstruction * aggregate_instructions, - Arena * arena) + Arena * arena) const { + /// `data_variants` will destroy the states of aggregate functions in the destructor + data_variants.aggregator = this; + data_variants.init(AggregatedDataVariants::Type::without_key); + + AggregatedDataWithoutKey & res = data_variants.without_key; + /// Adding values for (AggregateFunctionInstruction * inst = aggregate_instructions; inst->that; ++inst) { @@ -1622,15 +1629,32 @@ Block Aggregator::prepareBlockAndFill( } void Aggregator::addSingleKeyToAggregateColumns( - const AggregatedDataVariants & data_variants, + AggregatedDataVariants & data_variants, MutableColumns & aggregate_columns) const { - const auto & data = data_variants.without_key; - for (size_t i = 0; i < params.aggregates_size; ++i) + auto & data = data_variants.without_key; + + size_t i = 0; + try { - auto & column_aggregate_func = assert_cast(*aggregate_columns[i]); - column_aggregate_func.getData().push_back(data + offsets_of_aggregate_states[i]); + for (i = 0; i < params.aggregates_size; ++i) + { + auto & column_aggregate_func = assert_cast(*aggregate_columns[i]); + column_aggregate_func.getData().push_back(data + offsets_of_aggregate_states[i]); + } } + catch (...) + { + /// Rollback + for (size_t rollback_i = 0; rollback_i < i; ++rollback_i) + { + auto & column_aggregate_func = assert_cast(*aggregate_columns[rollback_i]); + column_aggregate_func.getData().pop_back(); + } + throw; + } + + data = nullptr; } void Aggregator::addArenasToAggregateColumns( @@ -2167,7 +2191,7 @@ ManyAggregatedDataVariants Aggregator::prepareVariantsToMerge(ManyAggregatedData if (non_empty_data.size() > 1) { /// Sort the states in descending order so that the merge is more efficient (since all states are merged into the first). - std::sort(non_empty_data.begin(), non_empty_data.end(), + ::sort(non_empty_data.begin(), non_empty_data.end(), [](const AggregatedDataVariantsPtr & lhs, const AggregatedDataVariantsPtr & rhs) { return lhs->sizeWithoutOverflowRow() > rhs->sizeWithoutOverflowRow(); diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index c79c2c5ef64..05c9133cb35 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -1138,12 +1138,12 @@ private: AggregateFunctionInstruction * aggregate_instructions, Arena * arena) const; - static void executeOnIntervalWithoutKeyImpl( - AggregatedDataWithoutKey & res, + void executeOnIntervalWithoutKeyImpl( + AggregatedDataVariants & data_variants, size_t row_begin, size_t row_end, AggregateFunctionInstruction * aggregate_instructions, - Arena * arena); + Arena * arena) const; template void writeToTemporaryFileImpl( @@ -1307,7 +1307,7 @@ private: NestedColumnsHolder & nested_columns_holder) const; void addSingleKeyToAggregateColumns( - const AggregatedDataVariants & data_variants, + AggregatedDataVariants & data_variants, MutableColumns & aggregate_columns) const; void addArenasToAggregateColumns( diff --git a/src/Interpreters/AsynchronousInsertQueue.cpp b/src/Interpreters/AsynchronousInsertQueue.cpp index 0e2605fa2e2..5321d5f6fd3 100644 --- a/src/Interpreters/AsynchronousInsertQueue.cpp +++ b/src/Interpreters/AsynchronousInsertQueue.cpp @@ -24,6 +24,16 @@ #include +namespace CurrentMetrics +{ + extern const Metric PendingAsyncInsert; +} + +namespace ProfileEvents +{ + extern const Event AsyncInsertQuery; +} + namespace DB { @@ -223,6 +233,9 @@ void AsynchronousInsertQueue::pushImpl(InsertData::EntryPtr entry, QueueIterator if (data->size > max_data_size) scheduleDataProcessingJob(it->first, std::move(data), getContext()); + + CurrentMetrics::add(CurrentMetrics::PendingAsyncInsert); + ProfileEvents::increment(ProfileEvents::AsyncInsertQuery); } void AsynchronousInsertQueue::waitForProcessingQuery(const String & query_id, const Milliseconds & timeout) @@ -437,6 +450,8 @@ try for (const auto & entry : data->entries) if (!entry->isFinished()) entry->finish(); + + CurrentMetrics::sub(CurrentMetrics::PendingAsyncInsert, data->entries.size()); } catch (const Exception & e) { @@ -470,6 +485,8 @@ void AsynchronousInsertQueue::finishWithException( entry->finish(std::make_exception_ptr(exception)); } } + + CurrentMetrics::sub(CurrentMetrics::PendingAsyncInsert, entries.size()); } } diff --git a/src/Interpreters/Cluster.cpp b/src/Interpreters/Cluster.cpp index 05972f2ee50..c01b19d81de 100644 --- a/src/Interpreters/Cluster.cpp +++ b/src/Interpreters/Cluster.cpp @@ -12,8 +12,10 @@ #include #include #include +#include #include + namespace DB { @@ -305,11 +307,11 @@ void Clusters::updateClusters(const Poco::Util::AbstractConfiguration & new_conf Poco::Util::AbstractConfiguration::Keys deleted_keys; if (old_config) { - std::sort(new_config_keys.begin(), new_config_keys.end()); + ::sort(new_config_keys.begin(), new_config_keys.end()); Poco::Util::AbstractConfiguration::Keys old_config_keys; old_config->keys(config_prefix, old_config_keys); - std::sort(old_config_keys.begin(), old_config_keys.end()); + ::sort(old_config_keys.begin(), old_config_keys.end()); std::set_difference( old_config_keys.begin(), old_config_keys.end(), new_config_keys.begin(), new_config_keys.end(), std::back_inserter(deleted_keys)); diff --git a/src/Interpreters/ClusterDiscovery.cpp b/src/Interpreters/ClusterDiscovery.cpp index 8b68ba02504..df6e8ea98f5 100644 --- a/src/Interpreters/ClusterDiscovery.cpp +++ b/src/Interpreters/ClusterDiscovery.cpp @@ -229,7 +229,7 @@ ClusterPtr ClusterDiscovery::makeCluster(const ClusterInfo & cluster_info) bool secure = cluster_info.current_node.secure; auto cluster = std::make_shared( - context->getSettings(), + context->getSettingsRef(), shards, /* username= */ context->getUserName(), /* password= */ "", diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp index a47874c475a..5d35525aee9 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.cpp @@ -35,11 +35,9 @@ namespace ClusterProxy SelectStreamFactory::SelectStreamFactory( const Block & header_, - QueryProcessingStage::Enum processed_stage_, - bool has_virtual_shard_num_column_) - : header(header_), - processed_stage{processed_stage_}, - has_virtual_shard_num_column(has_virtual_shard_num_column_) + QueryProcessingStage::Enum processed_stage_) + : header(header_) + , processed_stage{processed_stage_} { } @@ -102,19 +100,15 @@ void SelectStreamFactory::createForShard( Shards & remote_shards, UInt32 shard_count) { - auto modified_query_ast = query_ast->clone(); - if (has_virtual_shard_num_column) - VirtualColumnUtils::rewriteEntityInAst(modified_query_ast, "_shard_num", shard_info.shard_num, "toUInt32"); - auto emplace_local_stream = [&]() { - local_plans.emplace_back(createLocalPlan(modified_query_ast, header, context, processed_stage, shard_info.shard_num, shard_count)); + local_plans.emplace_back(createLocalPlan(query_ast, header, context, processed_stage, shard_info.shard_num, shard_count)); }; auto emplace_remote_stream = [&](bool lazy = false, UInt32 local_delay = 0) { remote_shards.emplace_back(Shard{ - .query = modified_query_ast, + .query = query_ast, .header = header, .shard_num = shard_info.shard_num, .num_replicas = shard_info.getAllNodeCount(), diff --git a/src/Interpreters/ClusterProxy/SelectStreamFactory.h b/src/Interpreters/ClusterProxy/SelectStreamFactory.h index dda6fb96f01..55e81feee33 100644 --- a/src/Interpreters/ClusterProxy/SelectStreamFactory.h +++ b/src/Interpreters/ClusterProxy/SelectStreamFactory.h @@ -16,8 +16,7 @@ class SelectStreamFactory final : public IStreamFactory public: SelectStreamFactory( const Block & header_, - QueryProcessingStage::Enum processed_stage_, - bool has_virtual_shard_num_column_); + QueryProcessingStage::Enum processed_stage_); void createForShard( const Cluster::ShardInfo & shard_info, @@ -32,8 +31,6 @@ public: private: const Block header; QueryProcessingStage::Enum processed_stage; - - bool has_virtual_shard_num_column = false; }; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 1c71ab2cd6f..e8115c384bb 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -208,6 +208,7 @@ struct ContextSharedPart mutable MarkCachePtr index_mark_cache; /// Cache of marks in compressed files of MergeTree indices. mutable MMappedFileCachePtr mmap_cache; /// Cache of mmapped files to avoid frequent open/map/unmap/close and to reuse from several threads. ProcessList process_list; /// Executing queries at the moment. + GlobalOvercommitTracker global_overcommit_tracker; MergeList merge_list; /// The list of executable merge (for (Replicated)?MergeTree) ReplicatedFetchList replicated_fetch_list; ConfigurationPtr users_config; /// Config with the users, profiles and quotas sections. @@ -275,7 +276,9 @@ struct ContextSharedPart Context::ConfigReloadCallback config_reload_callback; ContextSharedPart() - : access_control(std::make_unique()), macros(std::make_unique()) + : access_control(std::make_unique()) + , global_overcommit_tracker(&process_list) + , macros(std::make_unique()) { /// TODO: make it singleton (?) static std::atomic num_calls{0}; @@ -474,6 +477,7 @@ std::unique_lock Context::getLock() const ProcessList & Context::getProcessList() { return shared->process_list; } const ProcessList & Context::getProcessList() const { return shared->process_list; } +OvercommitTracker * Context::getGlobalOvercommitTracker() const { return &shared->global_overcommit_tracker; } MergeList & Context::getMergeList() { return shared->merge_list; } const MergeList & Context::getMergeList() const { return shared->merge_list; } ReplicatedFetchList & Context::getReplicatedFetchList() { return shared->replicated_fetch_list; } @@ -2650,6 +2654,19 @@ void Context::shutdown() } } + // Special volumes might also use disks that require shutdown. + for (const auto & volume : {shared->tmp_volume, shared->backups_volume}) + { + if (volume) + { + auto & disks = volume->getDisks(); + for (auto & disk : disks) + { + disk->shutdown(); + } + } + } + shared->shutdown(); } @@ -3135,6 +3152,7 @@ ReadSettings Context::getReadSettings() const res.http_max_tries = settings.http_max_tries; res.http_retry_initial_backoff_ms = settings.http_retry_initial_backoff_ms; res.http_retry_max_backoff_ms = settings.http_retry_max_backoff_ms; + res.http_skip_not_found_url_for_globs = settings.http_skip_not_found_url_for_globs; res.mmap_cache = getMMappedFileCache().get(); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 6b0a4671efb..2a2783603a2 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -29,6 +29,7 @@ namespace Poco::Net { class IPAddress; } namespace zkutil { class ZooKeeper; } +struct OvercommitTracker; namespace DB { @@ -657,6 +658,8 @@ public: ProcessList & getProcessList(); const ProcessList & getProcessList() const; + OvercommitTracker * getGlobalOvercommitTracker() const; + MergeList & getMergeList(); const MergeList & getMergeList() const; diff --git a/src/Interpreters/DDLTask.cpp b/src/Interpreters/DDLTask.cpp index 2499ce4d60d..64b9bf88ae9 100644 --- a/src/Interpreters/DDLTask.cpp +++ b/src/Interpreters/DDLTask.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -14,6 +15,7 @@ #include #include + namespace DB { @@ -324,7 +326,7 @@ String DDLTask::getShardID() const Strings replica_names; for (const Cluster::Address & address : shard_addresses) replica_names.emplace_back(address.readableString()); - std::sort(replica_names.begin(), replica_names.end()); + ::sort(replica_names.begin(), replica_names.end()); String res; for (auto it = replica_names.begin(); it != replica_names.end(); ++it) diff --git a/src/Interpreters/DDLWorker.cpp b/src/Interpreters/DDLWorker.cpp index 3eeb817cbab..3e414d5b6de 100644 --- a/src/Interpreters/DDLWorker.cpp +++ b/src/Interpreters/DDLWorker.cpp @@ -28,6 +28,7 @@ #include #include #include +#include #include #include #include @@ -221,7 +222,7 @@ DDLTaskPtr DDLWorker::initAndCheckTask(const String & entry_name, String & out_r static void filterAndSortQueueNodes(Strings & all_nodes) { all_nodes.erase(std::remove_if(all_nodes.begin(), all_nodes.end(), [] (const String & s) { return !startsWith(s, "query-"); }), all_nodes.end()); - std::sort(all_nodes.begin(), all_nodes.end()); + ::sort(all_nodes.begin(), all_nodes.end()); } void DDLWorker::scheduleTasks(bool reinitialized) diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp index 2117eec0063..ac8a27484d9 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.cpp @@ -1,9 +1,9 @@ #include -#include #include -#include +#include #include +#include #include #include #include @@ -18,7 +18,14 @@ #include #include #include +#include +namespace ProfileEvents +{ +extern const Event ScalarSubqueriesGlobalCacheHit; +extern const Event ScalarSubqueriesLocalCacheHit; +extern const Event ScalarSubqueriesCacheMiss; +} namespace DB { @@ -68,44 +75,99 @@ void ExecuteScalarSubqueriesMatcher::visit(ASTPtr & ast, Data & data) static bool worthConvertingToLiteral(const Block & scalar) { const auto * scalar_type_name = scalar.safeGetByPosition(0).type->getFamilyName(); - std::set useless_literal_types = {"Array", "Tuple", "AggregateFunction", "Function", "Set", "LowCardinality"}; + static const std::set useless_literal_types = {"Array", "Tuple", "AggregateFunction", "Function", "Set", "LowCardinality"}; return !useless_literal_types.count(scalar_type_name); } +static auto getQueryInterpreter(const ASTSubquery & subquery, ExecuteScalarSubqueriesMatcher::Data & data) +{ + auto subquery_context = Context::createCopy(data.getContext()); + Settings subquery_settings = data.getContext()->getSettings(); + subquery_settings.max_result_rows = 1; + subquery_settings.extremes = false; + subquery_context->setSettings(subquery_settings); + if (!data.only_analyze && subquery_context->hasQueryContext()) + { + /// Save current cached scalars in the context before analyzing the query + /// This is specially helpful when analyzing CTE scalars + auto context = subquery_context->getQueryContext(); + for (const auto & it : data.scalars) + context->addScalar(it.first, it.second); + } + + ASTPtr subquery_select = subquery.children.at(0); + + auto options = SelectQueryOptions(QueryProcessingStage::Complete, data.subquery_depth + 1, true); + options.analyze(data.only_analyze); + + return std::make_unique(subquery_select, subquery_context, options); +} + void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr & ast, Data & data) { auto hash = subquery.getTreeHash(); auto scalar_query_hash_str = toString(hash.first) + "_" + toString(hash.second); + std::unique_ptr interpreter = nullptr; + bool hit = false; + bool is_local = false; + Block scalar; - if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasScalar(scalar_query_hash_str)) + if (data.local_scalars.count(scalar_query_hash_str)) { - scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str); + hit = true; + scalar = data.local_scalars[scalar_query_hash_str]; + is_local = true; + ProfileEvents::increment(ProfileEvents::ScalarSubqueriesLocalCacheHit); } else if (data.scalars.count(scalar_query_hash_str)) { + hit = true; scalar = data.scalars[scalar_query_hash_str]; + ProfileEvents::increment(ProfileEvents::ScalarSubqueriesGlobalCacheHit); } else { - auto subquery_context = Context::createCopy(data.getContext()); - Settings subquery_settings = data.getContext()->getSettings(); - subquery_settings.max_result_rows = 1; - subquery_settings.extremes = false; - subquery_context->setSettings(subquery_settings); + if (data.getContext()->hasQueryContext() && data.getContext()->getQueryContext()->hasScalar(scalar_query_hash_str)) + { + if (!data.getContext()->getViewSource()) + { + /// We aren't using storage views so we can safely use the context cache + scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str); + ProfileEvents::increment(ProfileEvents::ScalarSubqueriesGlobalCacheHit); + hit = true; + } + else + { + /// If we are under a context that uses views that means that the cache might contain values that reference + /// the original table and not the view, so in order to be able to check the global cache we need to first + /// make sure that the query doesn't use the view + /// Note in any case the scalar will end up cached in *data* so this won't be repeated inside this context + interpreter = getQueryInterpreter(subquery, data); + if (!interpreter->usesViewSource()) + { + scalar = data.getContext()->getQueryContext()->getScalar(scalar_query_hash_str); + ProfileEvents::increment(ProfileEvents::ScalarSubqueriesGlobalCacheHit); + hit = true; + } + } + } + } - ASTPtr subquery_select = subquery.children.at(0); + if (!hit) + { + if (!interpreter) + interpreter = getQueryInterpreter(subquery, data); - auto options = SelectQueryOptions(QueryProcessingStage::Complete, data.subquery_depth + 1, true); - options.analyze(data.only_analyze); + ProfileEvents::increment(ProfileEvents::ScalarSubqueriesCacheMiss); + is_local = interpreter->usesViewSource(); - auto interpreter = InterpreterSelectWithUnionQuery(subquery_select, subquery_context, options); Block block; if (data.only_analyze) { /// If query is only analyzed, then constants are not correct. - block = interpreter.getSampleBlock(); + block = interpreter->getSampleBlock(); for (auto & column : block) { if (column.column->empty()) @@ -118,14 +180,14 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr } else { - auto io = interpreter.execute(); + auto io = interpreter->execute(); PullingAsyncPipelineExecutor executor(io.pipeline); while (block.rows() == 0 && executor.pull(block)); if (block.rows() == 0) { - auto types = interpreter.getSampleBlock().getDataTypes(); + auto types = interpreter->getSampleBlock().getDataTypes(); if (types.size() != 1) types = {std::make_shared(types)}; @@ -218,7 +280,10 @@ void ExecuteScalarSubqueriesMatcher::visit(const ASTSubquery & subquery, ASTPtr ast = std::move(func); } - data.scalars[scalar_query_hash_str] = std::move(scalar); + if (is_local) + data.local_scalars[scalar_query_hash_str] = std::move(scalar); + else + data.scalars[scalar_query_hash_str] = std::move(scalar); } void ExecuteScalarSubqueriesMatcher::visit(const ASTFunction & func, ASTPtr & ast, Data & data) diff --git a/src/Interpreters/ExecuteScalarSubqueriesVisitor.h b/src/Interpreters/ExecuteScalarSubqueriesVisitor.h index c230f346779..d702404dab6 100644 --- a/src/Interpreters/ExecuteScalarSubqueriesVisitor.h +++ b/src/Interpreters/ExecuteScalarSubqueriesVisitor.h @@ -19,11 +19,8 @@ struct ASTTableExpression; * * Features * - * A replacement occurs during query analysis, and not during the main runtime. - * This means that the progress indicator will not work during the execution of these requests, - * and also such queries can not be aborted. - * - * But the query result can be used for the index in the table. + * A replacement occurs during query analysis, and not during the main runtime, so + * the query result can be used for the index in the table. * * Scalar subqueries are executed on the request-initializer server. * The request is sent to remote servers with already substituted constants. @@ -37,6 +34,7 @@ public: { size_t subquery_depth; Scalars & scalars; + Scalars & local_scalars; bool only_analyze; }; diff --git a/src/Interpreters/ExpressionActions.cpp b/src/Interpreters/ExpressionActions.cpp index 73c191d39ab..30c832e4917 100644 --- a/src/Interpreters/ExpressionActions.cpp +++ b/src/Interpreters/ExpressionActions.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -158,8 +159,8 @@ static void setLazyExecutionInfo( const ActionsDAGReverseInfo::NodeInfo & node_info = reverse_info.nodes_info[reverse_info.reverse_index.at(node)]; - /// If node is used in result, we can't enable lazy execution. - if (node_info.used_in_result) + /// If node is used in result or it doesn't have parents, we can't enable lazy execution. + if (node_info.used_in_result || node_info.parents.empty()) lazy_execution_info.can_be_lazy_executed = false; /// To fill lazy execution info for current node we need to create it for all it's parents. @@ -735,7 +736,7 @@ void ExpressionActions::execute(Block & block, size_t & num_rows, bool dry_run) } else { - std::sort(execution_context.inputs_pos.rbegin(), execution_context.inputs_pos.rend()); + ::sort(execution_context.inputs_pos.rbegin(), execution_context.inputs_pos.rend()); for (auto input : execution_context.inputs_pos) if (input >= 0) block.erase(input); diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index c195cb93c5e..4a5f18a408f 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -944,7 +944,10 @@ static std::unique_ptr buildJoinedPlan( * - JOIN tables will need aliases to correctly resolve USING clause. */ auto interpreter = interpretSubquery( - join_element.table_expression, context, original_right_columns, query_options.copy().setWithAllColumns().ignoreAlias(false)); + join_element.table_expression, + context, + original_right_columns, + query_options.copy().setWithAllColumns().ignoreProjections(false).ignoreAlias(false)); auto joined_plan = std::make_unique(); interpreter->buildQueryPlan(*joined_plan); { diff --git a/src/Interpreters/ExpressionJIT.cpp b/src/Interpreters/ExpressionJIT.cpp index 90292d17fae..4f9375d3059 100644 --- a/src/Interpreters/ExpressionJIT.cpp +++ b/src/Interpreters/ExpressionJIT.cpp @@ -6,6 +6,7 @@ #include #include +#include #include #include #include @@ -576,7 +577,10 @@ void ActionsDAG::compileFunctions(size_t min_count_to_compile_expression, const /** Sort nodes before compilation using their children size to avoid compiling subexpression before compile parent expression. * This is needed to avoid compiling expression more than once with different names because of compilation order. */ - std::sort(nodes_to_compile.begin(), nodes_to_compile.end(), [&](const Node * lhs, const Node * rhs) { return node_to_data[lhs].children_size > node_to_data[rhs].children_size; }); + ::sort(nodes_to_compile.begin(), nodes_to_compile.end(), [&](const Node * lhs, const Node * rhs) + { + return node_to_data[lhs].children_size > node_to_data[rhs].children_size; + }); for (auto & node : nodes_to_compile) { diff --git a/src/Interpreters/ExternalLoader.cpp b/src/Interpreters/ExternalLoader.cpp index b2cd9495feb..aab3a9e7437 100644 --- a/src/Interpreters/ExternalLoader.cpp +++ b/src/Interpreters/ExternalLoader.cpp @@ -966,14 +966,14 @@ private: /// Does the loading, possibly in the separate thread. void doLoading(const String & name, size_t loading_id, bool forced_to_reload, size_t min_id_to_finish_loading_dependencies_, bool async, ThreadGroupStatusPtr thread_group = {}) { - if (thread_group) - CurrentThread::attachTo(thread_group); - SCOPE_EXIT_SAFE( if (thread_group) CurrentThread::detachQueryIfNotDetached(); ); + if (thread_group) + CurrentThread::attachTo(thread_group); + LOG_TRACE(log, "Start loading object '{}'", name); try { diff --git a/src/Interpreters/IInterpreterUnionOrSelectQuery.h b/src/Interpreters/IInterpreterUnionOrSelectQuery.h index db9cc086e35..1f59dd36354 100644 --- a/src/Interpreters/IInterpreterUnionOrSelectQuery.h +++ b/src/Interpreters/IInterpreterUnionOrSelectQuery.h @@ -40,6 +40,15 @@ public: void extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr &, ContextPtr) const override; + /// Returns whether the query uses the view source from the Context + /// The view source is a virtual storage that currently only materialized views use to replace the source table + /// with the incoming block only + /// This flag is useful to know for how long we can cache scalars generated by this query: If it doesn't use the virtual storage + /// then we can cache the scalars forever (for any query that doesn't use the virtual storage either), but if it does use the virtual + /// storage then we can only keep the scalar result around while we are working with that source block + /// You can find more details about this under ExecuteScalarSubqueriesMatcher::visit + bool usesViewSource() { return uses_view_source; } + protected: ASTPtr query_ptr; ContextMutablePtr context; @@ -48,6 +57,7 @@ protected: size_t max_streams = 1; bool settings_limit_offset_needed = false; bool settings_limit_offset_done = false; + bool uses_view_source = false; }; } diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index d2b77f1a439..71db15dc46f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -2,6 +2,7 @@ #include +#include "Common/Exception.h" #include #include #include @@ -12,6 +13,7 @@ #include #include +#include #include #include @@ -91,6 +93,7 @@ namespace ErrorCodes extern const int UNKNOWN_DATABASE; extern const int PATH_ACCESS_DENIED; extern const int NOT_IMPLEMENTED; + extern const int ENGINE_REQUIRED; } namespace fs = std::filesystem; @@ -157,6 +160,9 @@ BlockIO InterpreterCreateQuery::createDatabase(ASTCreateQuery & create) throw Exception(ErrorCodes::UNKNOWN_DATABASE_ENGINE, "Unknown database engine: {}", serializeAST(*create.storage)); } + if (create.storage && !create.storage->engine) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Database engine must be specified"); + if (create.storage->engine->name == "Atomic" || create.storage->engine->name == "Replicated" || create.storage->engine->name == "MaterializedPostgreSQL") @@ -581,6 +587,17 @@ ConstraintsDescription InterpreterCreateQuery::getConstraintsDescription(const A InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create) const { + /// Set the table engine if it was not specified explicitly. + setEngine(create); + + /// We have to check access rights again (in case engine was changed). + if (create.storage) + { + auto source_access_type = StorageFactory::instance().getSourceAccessType(create.storage->engine->name); + if (source_access_type != AccessType::NONE) + getContext()->checkAccess(source_access_type); + } + TableProperties properties; TableLockHolder as_storage_lock; @@ -645,7 +662,7 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti } /// We can have queries like "CREATE TABLE
Test nameTest statusDescription
ENGINE=" if /// supports schema inference (will determine table structure in it's constructor). - else if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(create.storage->engine->name)) + else if (!StorageFactory::instance().checkIfStorageSupportsSchemaInterface(create.storage->engine->name)) // NOLINT throw Exception("Incorrect CREATE query: required list of column descriptions or AS section or SELECT.", ErrorCodes::INCORRECT_QUERY); /// Even if query has list of columns, canonicalize it (unfold Nested columns). @@ -663,8 +680,6 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti create.columns_list->setOrReplace(create.columns_list->projections, new_projections); validateTableStructure(create, properties); - /// Set the table engine if it was not specified explicitly. - setEngine(create); assert(as_database_saved.empty() && as_table_saved.empty()); std::swap(create.as_database, as_database_saved); @@ -718,30 +733,90 @@ void InterpreterCreateQuery::validateTableStructure(const ASTCreateQuery & creat } } +String InterpreterCreateQuery::getTableEngineName(DefaultTableEngine default_table_engine) +{ + switch (default_table_engine) + { + case DefaultTableEngine::Log: + return "Log"; + + case DefaultTableEngine::StripeLog: + return "StripeLog"; + + case DefaultTableEngine::MergeTree: + return "MergeTree"; + + case DefaultTableEngine::ReplacingMergeTree: + return "ReplacingMergeTree"; + + case DefaultTableEngine::ReplicatedMergeTree: + return "ReplicatedMergeTree"; + + case DefaultTableEngine::ReplicatedReplacingMergeTree: + return "ReplicatedReplacingMergeTree"; + + case DefaultTableEngine::Memory: + return "Memory"; + + default: + throw Exception("default_table_engine is set to unknown value", ErrorCodes::LOGICAL_ERROR); + } +} + +void InterpreterCreateQuery::setDefaultTableEngine(ASTStorage & storage, ContextPtr local_context) +{ + if (local_context->getSettingsRef().default_table_engine.value == DefaultTableEngine::None) + throw Exception(ErrorCodes::ENGINE_REQUIRED, "Table engine is not specified in CREATE query"); + + auto engine_ast = std::make_shared(); + auto default_table_engine = local_context->getSettingsRef().default_table_engine.value; + engine_ast->name = getTableEngineName(default_table_engine); + engine_ast->no_empty_args = true; + storage.set(storage.engine, engine_ast); +} + void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const { if (create.as_table_function) return; - if (create.storage || create.is_dictionary || create.isView()) - { - if (create.temporary && create.storage && create.storage->engine && create.storage->engine->name != "Memory") - throw Exception(ErrorCodes::INCORRECT_QUERY, - "Temporary tables can only be created with ENGINE = Memory, not {}", create.storage->engine->name); - + if (create.is_dictionary || create.is_ordinary_view || create.is_live_view || create.is_window_view) + return; + + if (create.is_materialized_view && create.to_table_id) return; - } if (create.temporary) { + if (create.storage && create.storage->engine && create.storage->engine->name != "Memory") + throw Exception(ErrorCodes::INCORRECT_QUERY, "Temporary tables can only be created with ENGINE = Memory, not {}", + create.storage->engine->name); + + /// It's possible if some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not. + /// It makes sense when default_table_engine setting is used, but not for temporary tables. + /// For temporary tables we ignore this setting to allow CREATE TEMPORARY TABLE query without specifying ENGINE + /// even if setting is set to MergeTree or something like that (otherwise MergeTree will be substituted and query will fail). + if (create.storage && !create.storage->engine) + throw Exception(ErrorCodes::INCORRECT_QUERY, "Invalid storage definition for temporary table: must be either ENGINE = Memory or empty"); + auto engine_ast = std::make_shared(); engine_ast->name = "Memory"; engine_ast->no_empty_args = true; auto storage_ast = std::make_shared(); storage_ast->set(storage_ast->engine, engine_ast); create.set(create.storage, storage_ast); + return; } - else if (!create.as_table.empty()) + + if (create.storage) + { + /// Some part of storage definition (such as PARTITION BY) is specified, but ENGINE is not: just set default one. + if (!create.storage->engine) + setDefaultTableEngine(*create.storage, getContext()); + return; + } + + if (!create.as_table.empty()) { /// NOTE Getting the structure from the table specified in the AS is done not atomically with the creation of the table. @@ -754,24 +829,16 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const const String qualified_name = backQuoteIfNeed(as_database_name) + "." + backQuoteIfNeed(as_table_name); if (as_create.is_ordinary_view) - throw Exception( - "Cannot CREATE a table AS " + qualified_name + ", it is a View", - ErrorCodes::INCORRECT_QUERY); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a View", qualified_name); if (as_create.is_live_view) - throw Exception( - "Cannot CREATE a table AS " + qualified_name + ", it is a Live View", - ErrorCodes::INCORRECT_QUERY); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Live View", qualified_name); if (as_create.is_window_view) - throw Exception( - "Cannot CREATE a table AS " + qualified_name + ", it is a Window View", - ErrorCodes::INCORRECT_QUERY); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Window View", qualified_name); if (as_create.is_dictionary) - throw Exception( - "Cannot CREATE a table AS " + qualified_name + ", it is a Dictionary", - ErrorCodes::INCORRECT_QUERY); + throw Exception(ErrorCodes::INCORRECT_QUERY, "Cannot CREATE a table AS {}, it is a Dictionary", qualified_name); if (as_create.storage) create.set(create.storage, as_create.storage->ptr()); @@ -779,7 +846,12 @@ void InterpreterCreateQuery::setEngine(ASTCreateQuery & create) const create.as_table_function = as_create.as_table_function->clone(); else throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot set engine, it's a bug."); + + return; } + + create.set(create.storage, std::make_shared()); + setDefaultTableEngine(*create.storage, getContext()); } static void generateUUIDForTable(ASTCreateQuery & create) @@ -1034,6 +1106,20 @@ bool InterpreterCreateQuery::doCreateTable(ASTCreateQuery & create, throw Exception(storage_already_exists_error_code, "{} {}.{} already exists", storage_name, backQuoteIfNeed(create.getDatabase()), backQuoteIfNeed(create.getTable())); } + else if (!create.attach) + { + /// Checking that table may exists in detached/detached permanently state + try + { + database->checkMetadataFilenameAvailability(create.getTable()); + } + catch (const Exception &) + { + if (create.if_not_exists) + return false; + throw; + } + } data_path = database->getTableDataPath(create); diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 03c4b4ae1b6..5804d817fe2 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -15,6 +16,7 @@ namespace DB class ASTCreateQuery; class ASTExpressionList; class ASTConstraintDeclaration; +class ASTStorage; class IDatabase; using DatabasePtr = std::shared_ptr; @@ -81,6 +83,8 @@ private: /// Calculate list of columns, constraints, indices, etc... of table. Rewrite query in canonical way. TableProperties getTablePropertiesAndNormalizeCreateQuery(ASTCreateQuery & create) const; void validateTableStructure(const ASTCreateQuery & create, const TableProperties & properties) const; + static String getTableEngineName(DefaultTableEngine default_table_engine); + static void setDefaultTableEngine(ASTStorage & storage, ContextPtr local_context); void setEngine(ASTCreateQuery & create) const; AccessRightsElements getRequiredAccess() const; diff --git a/src/Interpreters/InterpreterDropQuery.cpp b/src/Interpreters/InterpreterDropQuery.cpp index 90c4311b032..b04342b37f7 100644 --- a/src/Interpreters/InterpreterDropQuery.cpp +++ b/src/Interpreters/InterpreterDropQuery.cpp @@ -35,6 +35,10 @@ namespace ErrorCodes extern const int TABLE_IS_READ_ONLY; } +namespace ActionLocks +{ + extern const StorageActionBlockType PartsMerge; +} static DatabasePtr tryGetDatabase(const String & database_name, bool if_exists) { @@ -202,7 +206,15 @@ BlockIO InterpreterDropQuery::executeToTableImpl(ContextPtr context_, ASTDropQue table->checkTableCanBeDropped(); - auto table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + TableExclusiveLockHolder table_lock; + /// We don't need this lock for ReplicatedMergeTree + if (!table->supportsReplication()) + { + /// And for simple MergeTree we can stop merges before acquiring the lock + auto merges_blocker = table->getActionLock(ActionLocks::PartsMerge); + auto table_lock = table->lockExclusively(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + } + auto metadata_snapshot = table->getInMemoryMetadataPtr(); /// Drop table data, don't touch metadata table->truncate(query_ptr, metadata_snapshot, context_, table_lock); diff --git a/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp b/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp index 3bb78b57702..cad570ab420 100644 --- a/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp +++ b/src/Interpreters/InterpreterSelectIntersectExceptQuery.cpp @@ -68,7 +68,10 @@ InterpreterSelectIntersectExceptQuery::InterpreterSelectIntersectExceptQuery( nested_interpreters.resize(num_children); for (size_t i = 0; i < num_children; ++i) + { nested_interpreters[i] = buildCurrentChildInterpreter(children.at(i)); + uses_view_source |= nested_interpreters[i]->usesViewSource(); + } Blocks headers(num_children); for (size_t query_num = 0; query_num < num_children; ++query_num) diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 8e0f73f0b31..ac807f4f782 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -64,14 +64,16 @@ #include #include -#include #include +#include +#include #include #include #include #include #include +#include #include #include #include @@ -314,6 +316,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( if (!has_input && !storage) { storage = joined_tables.getLeftTableStorage(); + // Mark uses_view_source if the returned storage is the same as the one saved in viewSource + uses_view_source |= storage && storage == context->getViewSource(); got_storage_from_query = true; } @@ -335,6 +339,15 @@ InterpreterSelectQuery::InterpreterSelectQuery( joined_tables.reset(getSelectQuery()); joined_tables.resolveTables(); + if (auto view_source = context->getViewSource()) + { + // If we are using a virtual block view to replace a table and that table is used + // inside the JOIN then we need to update uses_view_source accordingly so we avoid propagating scalars that we can't cache + const auto & storage_values = static_cast(*view_source); + auto tmp_table_id = storage_values.getStorageID(); + for (const auto & t : joined_tables.tablesWithColumns()) + uses_view_source |= (t.table.database == tmp_table_id.database_name && t.table.table == tmp_table_id.table_name); + } if (storage && joined_tables.isLeftTableSubquery()) { @@ -350,7 +363,10 @@ InterpreterSelectQuery::InterpreterSelectQuery( { interpreter_subquery = joined_tables.makeLeftTableSubquery(options.subquery()); if (interpreter_subquery) + { source_header = interpreter_subquery->getSampleBlock(); + uses_view_source |= interpreter_subquery->usesViewSource(); + } } joined_tables.rewriteDistributedInAndJoins(query_ptr); @@ -388,9 +404,8 @@ InterpreterSelectQuery::InterpreterSelectQuery( query.setFinal(); /// Save scalar sub queries's results in the query context - /// But discard them if the Storage has been modified - /// In an ideal situation we would only discard the scalars affected by the storage change - if (!options.only_analyze && context->hasQueryContext() && !context->getViewSource()) + /// Note that we are only saving scalars and not local_scalars since the latter can't be safely shared across contexts + if (!options.only_analyze && context->hasQueryContext()) for (const auto & it : syntax_analyzer_result->getScalars()) context->getQueryContext()->addScalar(it.first, it.second); @@ -478,6 +493,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( /// If there is an aggregation in the outer query, WITH TOTALS is ignored in the subquery. if (query_analyzer->hasAggregation()) interpreter_subquery->ignoreWithTotals(); + uses_view_source |= interpreter_subquery->usesViewSource(); } required_columns = syntax_analyzer_result->requiredSourceColumns(); @@ -1179,8 +1195,10 @@ void InterpreterSelectQuery::executeImpl(QueryPlan & query_plan, std::optional

input_order_info.reset(); } // Now we must execute: @@ -1928,6 +1946,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc { query_info.projection->order_optimizer = std::make_shared( // TODO Do we need a projection variant for this field? + query, analysis_result.order_by_elements_actions, getSortDescription(query, context), query_info.syntax_analyzer_result); @@ -1935,7 +1954,10 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc else { query_info.order_optimizer = std::make_shared( - analysis_result.order_by_elements_actions, getSortDescription(query, context), query_info.syntax_analyzer_result); + query, + analysis_result.order_by_elements_actions, + getSortDescription(query, context), + query_info.syntax_analyzer_result); } } else @@ -1943,14 +1965,18 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc if (query_info.projection) { query_info.projection->order_optimizer = std::make_shared( + query, query_info.projection->group_by_elements_actions, - getSortDescriptionFromGroupBy(query), + query_info.projection->group_by_elements_order_descr, query_info.syntax_analyzer_result); } else { query_info.order_optimizer = std::make_shared( - analysis_result.group_by_elements_actions, getSortDescriptionFromGroupBy(query), query_info.syntax_analyzer_result); + query, + analysis_result.group_by_elements_actions, + getSortDescriptionFromGroupBy(query), + query_info.syntax_analyzer_result); } } @@ -2269,7 +2295,7 @@ void InterpreterSelectQuery::executeWindow(QueryPlan & query_plan) for (const auto & [_, w] : query_analyzer->windowDescriptions()) windows_sorted.push_back(&w); - std::sort(windows_sorted.begin(), windows_sorted.end(), windowDescriptionComparator); + ::sort(windows_sorted.begin(), windows_sorted.end(), windowDescriptionComparator); const Settings & settings = context->getSettingsRef(); for (size_t i = 0; i < windows_sorted.size(); ++i) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index e4b3e62c358..723db59f04b 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -138,6 +138,9 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( nested_interpreters.emplace_back( buildCurrentChildInterpreter(ast->list_of_selects->children.at(query_num), require_full_header ? Names() : current_required_result_column_names)); + // We need to propagate the uses_view_source flag from children to the (self) parent since, if one of the children uses + // a view source that means that the parent uses it too and can be cached globally + uses_view_source |= nested_interpreters.back()->usesViewSource(); } /// Determine structure of the result. diff --git a/src/Interpreters/LogicalExpressionsOptimizer.cpp b/src/Interpreters/LogicalExpressionsOptimizer.cpp index 936ed0149d2..d1cec92ceb7 100644 --- a/src/Interpreters/LogicalExpressionsOptimizer.cpp +++ b/src/Interpreters/LogicalExpressionsOptimizer.cpp @@ -9,6 +9,8 @@ #include +#include + namespace DB { @@ -180,7 +182,7 @@ void LogicalExpressionsOptimizer::collectDisjunctiveEqualityChains() { auto & equalities = chain.second; auto & equality_functions = equalities.functions; - std::sort(equality_functions.begin(), equality_functions.end()); + ::sort(equality_functions.begin(), equality_functions.end()); } } @@ -237,7 +239,7 @@ void LogicalExpressionsOptimizer::addInExpression(const DisjunctiveEqualityChain } /// Sort the literals so that they are specified in the same order in the IN expression. - std::sort(tuple.begin(), tuple.end()); + ::sort(tuple.begin(), tuple.end()); /// Get the expression `expr` from the chain `expr = x1 OR ... OR expr = xN` ASTPtr equals_expr_lhs; diff --git a/src/Interpreters/OpenTelemetrySpanLog.cpp b/src/Interpreters/OpenTelemetrySpanLog.cpp index 10de6ba0e7b..40f31e4976c 100644 --- a/src/Interpreters/OpenTelemetrySpanLog.cpp +++ b/src/Interpreters/OpenTelemetrySpanLog.cpp @@ -150,24 +150,6 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder() } } - -template -static T readHex(const char * data) -{ - T x{}; - - const char * end = data + sizeof(T) * 2; - while (data < end) - { - x *= 16; - x += unhex(*data); - ++data; - } - - return x; -} - - bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & traceparent, std::string & error) { @@ -185,7 +167,7 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace const char * data = traceparent.data(); - uint8_t version = readHex(data); + uint8_t version = unhex2(data); data += 2; if (version != 0) @@ -201,7 +183,8 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace } ++data; - UInt128 trace_id_128 = readHex(data); + UInt64 trace_id_higher_64 = unhexUInt(data); + UInt64 trace_id_lower_64 = unhexUInt(data + 16); data += 32; if (*data != '-') @@ -211,7 +194,7 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace } ++data; - UInt64 span_id_64 = readHex(data); + UInt64 span_id_64 = unhexUInt(data); data += 16; if (*data != '-') @@ -221,8 +204,11 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace } ++data; - this->trace_flags = readHex(data); - this->trace_id = trace_id_128; + this->trace_flags = unhex2(data); + + // store the 128-bit trace id in big-endian order + this->trace_id.toUnderType().items[0] = trace_id_higher_64; + this->trace_id.toUnderType().items[1] = trace_id_lower_64; this->span_id = span_id_64; return true; } @@ -232,11 +218,14 @@ std::string OpenTelemetryTraceContext::composeTraceparentHeader() const { // This span is a parent for its children, so we specify this span_id as a // parent id. - return fmt::format("00-{:032x}-{:016x}-{:02x}", __uint128_t(trace_id.toUnderType()), - span_id, - // This cast is needed because fmt is being weird and complaining that - // "mixing character types is not allowed". - static_cast(trace_flags)); + return fmt::format("00-{:016x}{:016x}-{:016x}-{:02x}", + // Output the trace id in network byte order + trace_id.toUnderType().items[0], + trace_id.toUnderType().items[1], + span_id, + // This cast is needed because fmt is being weird and complaining that + // "mixing character types is not allowed". + static_cast(trace_flags)); } diff --git a/src/Interpreters/ProcessList.cpp b/src/Interpreters/ProcessList.cpp index 37b2992d657..d17d8e4972a 100644 --- a/src/Interpreters/ProcessList.cpp +++ b/src/Interpreters/ProcessList.cpp @@ -17,6 +17,11 @@ #include +namespace CurrentMetrics +{ + extern const Metric Query; +} + namespace DB { @@ -190,33 +195,21 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as ErrorCodes::QUERY_WITH_SAME_ID_IS_ALREADY_RUNNING); } - auto process_it = processes.emplace(processes.end(), - query_context, query_, client_info, priorities.insert(settings.priority), query_kind); - - increaseQueryKindAmount(query_kind); - - res = std::make_shared(*this, process_it); - ProcessListForUser & user_process_list = user_to_queries[client_info.current_user]; - user_process_list.queries.emplace(client_info.current_query_id, &res->get()); - - process_it->setUserProcessList(&user_process_list); - - /// Track memory usage for all simultaneously running queries from single user. - user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); - user_process_list.user_memory_tracker.setDescription("(for user)"); /// Actualize thread group info - if (auto thread_group = CurrentThread::getGroup()) + auto thread_group = CurrentThread::getGroup(); + if (thread_group) { std::lock_guard lock_thread_group(thread_group->mutex); thread_group->performance_counters.setParent(&user_process_list.user_performance_counters); thread_group->memory_tracker.setParent(&user_process_list.user_memory_tracker); - thread_group->query = process_it->query; - thread_group->normalized_query_hash = normalizedQueryHash(process_it->query); + thread_group->query = query_; + thread_group->normalized_query_hash = normalizedQueryHash(query_); /// Set query-level memory trackers thread_group->memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage); + thread_group->memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage); if (query_context->hasTraceCollector()) { @@ -231,10 +224,28 @@ ProcessList::EntryPtr ProcessList::insert(const String & query_, const IAST * as /// NOTE: Do not set the limit for thread-level memory tracker since it could show unreal values /// since allocation and deallocation could happen in different threads - - process_it->thread_group = std::move(thread_group); } + auto process_it = processes.emplace(processes.end(), + query_context, query_, client_info, priorities.insert(settings.priority), std::move(thread_group), query_kind); + + increaseQueryKindAmount(query_kind); + + res = std::make_shared(*this, process_it); + + process_it->setUserProcessList(&user_process_list); + + { + BlockQueryIfMemoryLimit block_query{user_process_list.user_overcommit_tracker}; + user_process_list.queries.emplace(client_info.current_query_id, &res->get()); + } + + /// Track memory usage for all simultaneously running queries from single user. + user_process_list.user_memory_tracker.setOrRaiseHardLimit(settings.max_memory_usage_for_user); + user_process_list.user_memory_tracker.setSoftLimit(settings.max_guaranteed_memory_usage_for_user); + user_process_list.user_memory_tracker.setDescription("(for user)"); + user_process_list.user_overcommit_tracker.setMaxWaitTime(settings.memory_usage_overcommit_max_wait_microseconds); + if (!user_process_list.user_throttler) { if (settings.max_network_bandwidth_for_user) @@ -263,9 +274,6 @@ ProcessListEntry::~ProcessListEntry() const QueryStatus * process_list_element_ptr = &*it; - /// This removes the memory_tracker of one request. - parent.processes.erase(it); - auto user_process_list_it = parent.user_to_queries.find(user); if (user_process_list_it == parent.user_to_queries.end()) { @@ -281,11 +289,15 @@ ProcessListEntry::~ProcessListEntry() { if (running_query->second == process_list_element_ptr) { + BlockQueryIfMemoryLimit block_query{user_process_list.user_overcommit_tracker}; user_process_list.queries.erase(running_query->first); found = true; } } + /// This removes the memory_tracker of one request. + parent.processes.erase(it); + if (!found) { LOG_ERROR(&Poco::Logger::get("ProcessList"), "Logical error: cannot find query by query_id and pointer to ProcessListElement in ProcessListForUser"); @@ -307,12 +319,19 @@ ProcessListEntry::~ProcessListEntry() QueryStatus::QueryStatus( - ContextPtr context_, const String & query_, const ClientInfo & client_info_, QueryPriorities::Handle && priority_handle_, IAST::QueryKind query_kind_) + ContextPtr context_, + const String & query_, + const ClientInfo & client_info_, + QueryPriorities::Handle && priority_handle_, + ThreadGroupStatusPtr && thread_group_, + IAST::QueryKind query_kind_) : WithContext(context_) , query(query_) , client_info(client_info_) + , thread_group(std::move(thread_group_)) , priority_handle(std::move(priority_handle_)) , query_kind(query_kind_) + , num_queries_increment(CurrentMetrics::Query) { auto settings = getContext()->getSettings(); limits.max_execution_time = settings.max_execution_time; @@ -322,6 +341,14 @@ QueryStatus::QueryStatus( QueryStatus::~QueryStatus() { assert(executors.empty()); + + if (auto * memory_tracker = getMemoryTracker()) + { + if (user_process_list) + user_process_list->user_overcommit_tracker.unsubscribe(memory_tracker); + if (auto shared_context = getContext()) + shared_context->getGlobalOvercommitTracker()->unsubscribe(memory_tracker); + } } CancellationCode QueryStatus::cancelQuery(bool) @@ -475,7 +502,11 @@ ProcessList::Info ProcessList::getInfo(bool get_thread_list, bool get_profile_ev } -ProcessListForUser::ProcessListForUser() = default; +ProcessListForUser::ProcessListForUser() + : user_overcommit_tracker(this) +{ + user_memory_tracker.setOvercommitTracker(&user_overcommit_tracker); +} ProcessListForUserInfo ProcessListForUser::getInfo(bool get_profile_events) const diff --git a/src/Interpreters/ProcessList.h b/src/Interpreters/ProcessList.h index 545e5b07345..5fb4bca7e13 100644 --- a/src/Interpreters/ProcessList.h +++ b/src/Interpreters/ProcessList.h @@ -16,6 +16,7 @@ #include #include #include +#include #include #include @@ -76,6 +77,7 @@ protected: friend class ThreadStatus; friend class CurrentThread; friend class ProcessListEntry; + friend struct ::GlobalOvercommitTracker; String query; ClientInfo client_info; @@ -121,6 +123,10 @@ protected: IAST::QueryKind query_kind; + /// This field is unused in this class, but it + /// increments/decrements metric in constructor/destructor. + CurrentMetrics::Increment num_queries_increment; + public: QueryStatus( @@ -128,6 +134,7 @@ public: const String & query_, const ClientInfo & client_info_, QueryPriorities::Handle && priority_handle_, + ThreadGroupStatusPtr && thread_group_, IAST::QueryKind query_kind_ ); @@ -150,6 +157,13 @@ public: ThrottlerPtr getUserNetworkThrottler(); + MemoryTracker * getMemoryTracker() const + { + if (!thread_group) + return nullptr; + return &thread_group->memory_tracker; + } + bool updateProgressIn(const Progress & value) { CurrentThread::updateProgressIn(value); @@ -212,6 +226,8 @@ struct ProcessListForUser /// Limit and counter for memory of all simultaneously running queries of single user. MemoryTracker user_memory_tracker{VariableContext::User}; + UserOvercommitTracker user_overcommit_tracker; + /// Count network usage for all simultaneously running queries of single user. ThrottlerPtr user_throttler; @@ -333,6 +349,14 @@ public: max_size = max_size_; } + template + void processEachQueryStatus(F && func) const + { + std::lock_guard lk(mutex); + for (auto && query : processes) + func(query); + } + void setMaxInsertQueriesAmount(size_t max_insert_queries_amount_) { std::lock_guard lock(mutex); diff --git a/src/Interpreters/RowRefs.h b/src/Interpreters/RowRefs.h index 987fd197d9d..cae20b98caf 100644 --- a/src/Interpreters/RowRefs.h +++ b/src/Interpreters/RowRefs.h @@ -1,15 +1,18 @@ #pragma once -#include -#include -#include - #include #include #include #include #include +#include + +#include +#include +#include + + namespace DB { @@ -199,7 +202,7 @@ private: if (!sorted.load(std::memory_order_relaxed)) { if (!array.empty()) - std::sort(array.begin(), array.end(), (ascending ? less : greater)); + ::sort(array.begin(), array.end(), (ascending ? less : greater)); sorted.store(true, std::memory_order_release); } diff --git a/src/Interpreters/Set.cpp b/src/Interpreters/Set.cpp index 0ccaae9a795..32dac7f9e9b 100644 --- a/src/Interpreters/Set.cpp +++ b/src/Interpreters/Set.cpp @@ -25,8 +25,10 @@ #include #include +#include #include + namespace DB { @@ -405,7 +407,7 @@ void Set::checkTypesEqual(size_t set_type_idx, const DataTypePtr & other_type) c MergeTreeSetIndex::MergeTreeSetIndex(const Columns & set_elements, std::vector && indexes_mapping_) : has_all_keys(set_elements.size() == indexes_mapping_.size()), indexes_mapping(std::move(indexes_mapping_)) { - std::sort(indexes_mapping.begin(), indexes_mapping.end(), + ::sort(indexes_mapping.begin(), indexes_mapping.end(), [](const KeyTuplePositionMapping & l, const KeyTuplePositionMapping & r) { return std::tie(l.key_index, l.tuple_index) < std::tie(r.key_index, r.tuple_index); diff --git a/src/Interpreters/SortedBlocksWriter.cpp b/src/Interpreters/SortedBlocksWriter.cpp index 1945824636f..3caf144d9a8 100644 --- a/src/Interpreters/SortedBlocksWriter.cpp +++ b/src/Interpreters/SortedBlocksWriter.cpp @@ -320,25 +320,7 @@ Block SortedBlocksBuffer::mergeBlocks(Blocks && blocks) const if (blocks.size() == 1) return blocks[0]; - Block out = blocks[0].cloneEmpty(); - - { /// Concatenate blocks - MutableColumns columns = out.mutateColumns(); - - for (size_t i = 0; i < columns.size(); ++i) - { - columns[i]->reserve(num_rows); - for (const auto & block : blocks) - { - const auto & tmp_column = *block.getByPosition(i).column; - columns[i]->insertRangeFrom(tmp_column, 0, block.rows()); - } - } - - out.setColumns(std::move(columns)); - } - - return out; + return concatenateBlocks(blocks); } } diff --git a/src/Interpreters/TreeCNFConverter.cpp b/src/Interpreters/TreeCNFConverter.cpp index e8e6ce09804..1f61c88ddd0 100644 --- a/src/Interpreters/TreeCNFConverter.cpp +++ b/src/Interpreters/TreeCNFConverter.cpp @@ -27,7 +27,7 @@ bool isLogicalFunction(const ASTFunction & func) size_t countAtoms(const ASTPtr & node) { checkStackSize(); - if (node->as()) + if (node->as() || node->as()) return 1; const auto * func = node->as(); diff --git a/src/Interpreters/TreeRewriter.cpp b/src/Interpreters/TreeRewriter.cpp index 0285bdf333c..fc3ef681c2c 100644 --- a/src/Interpreters/TreeRewriter.cpp +++ b/src/Interpreters/TreeRewriter.cpp @@ -479,10 +479,11 @@ void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query, } /// Replacing scalar subqueries with constant values. -void executeScalarSubqueries(ASTPtr & query, ContextPtr context, size_t subquery_depth, Scalars & scalars, bool only_analyze) +void executeScalarSubqueries( + ASTPtr & query, ContextPtr context, size_t subquery_depth, Scalars & scalars, Scalars & local_scalars, bool only_analyze) { LogAST log; - ExecuteScalarSubqueriesVisitor::Data visitor_data{WithContext{context}, subquery_depth, scalars, only_analyze}; + ExecuteScalarSubqueriesVisitor::Data visitor_data{WithContext{context}, subquery_depth, scalars, local_scalars, only_analyze}; ExecuteScalarSubqueriesVisitor(visitor_data, log.stream()).visit(query); } @@ -792,6 +793,39 @@ void markTupleLiteralsAsLegacy(ASTPtr & query) MarkTupleLiteralsAsLegacyVisitor(data).visit(query); } +/// Rewrite _shard_num -> shardNum() AS _shard_num +struct RewriteShardNum +{ + struct Data + { + }; + + static bool needChildVisit(const ASTPtr & parent, const ASTPtr & /*child*/) + { + /// ON section should not be rewritten. + return typeid_cast(parent.get()) == nullptr; + } + + static void visit(ASTPtr & ast, Data &) + { + if (auto * identifier = typeid_cast(ast.get())) + visit(*identifier, ast); + } + + static void visit(ASTIdentifier & identifier, ASTPtr & ast) + { + if (identifier.shortName() != "_shard_num") + return; + + String alias = identifier.tryGetAlias(); + if (alias.empty()) + alias = "_shard_num"; + ast = makeASTFunction("shardNum"); + ast->setAlias(alias); + } +}; +using RewriteShardNumVisitor = InDepthNodeVisitor; + } TreeRewriterResult::TreeRewriterResult( @@ -962,6 +996,7 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select ++it; } + has_virtual_shard_num = false; /// If there are virtual columns among the unknown columns. Remove them from the list of unknown and add /// in columns list, so that when further processing they are also considered. if (storage) @@ -978,6 +1013,18 @@ void TreeRewriterResult::collectUsedColumns(const ASTPtr & query, bool is_select else ++it; } + + if (is_remote_storage) + { + for (const auto & name_type : storage_virtuals) + { + if (name_type.name == "_shard_num" && storage->isVirtualColumn("_shard_num", metadata_snapshot)) + { + has_virtual_shard_num = true; + break; + } + } + } } if (!unknown_required_source_columns.empty()) @@ -1112,7 +1159,7 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( removeUnneededColumnsFromSelectClause(select_query, required_result_columns, remove_duplicates); /// Executing scalar subqueries - replacing them with constant values. - executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, select_options.only_analyze); + executeScalarSubqueries(query, getContext(), subquery_depth, result.scalars, result.local_scalars, select_options.only_analyze); if (settings.legacy_column_name_of_tuple_literal) markTupleLiteralsAsLegacy(query); @@ -1165,6 +1212,13 @@ TreeRewriterResultPtr TreeRewriter::analyzeSelect( } } + /// Rewrite _shard_num to shardNum() + if (result.has_virtual_shard_num) + { + RewriteShardNumVisitor::Data data_rewrite_shard_num; + RewriteShardNumVisitor(data_rewrite_shard_num).visit(query); + } + result.ast_join = select_query->join(); if (result.optimize_trivial_count) @@ -1195,7 +1249,7 @@ TreeRewriterResultPtr TreeRewriter::analyze( normalize(query, result.aliases, result.source_columns_set, false, settings, allow_self_aliases); /// Executing scalar subqueries. Column defaults could be a scalar subquery. - executeScalarSubqueries(query, getContext(), 0, result.scalars, !execute_scalar_subqueries); + executeScalarSubqueries(query, getContext(), 0, result.scalars, result.local_scalars, !execute_scalar_subqueries); if (settings.legacy_column_name_of_tuple_literal) markTupleLiteralsAsLegacy(query); diff --git a/src/Interpreters/TreeRewriter.h b/src/Interpreters/TreeRewriter.h index 52c62cc4cec..45b3a5a00e3 100644 --- a/src/Interpreters/TreeRewriter.h +++ b/src/Interpreters/TreeRewriter.h @@ -70,8 +70,12 @@ struct TreeRewriterResult /// Cache isRemote() call for storage, because it may be too heavy. bool is_remote_storage = false; + /// Rewrite _shard_num to shardNum() + bool has_virtual_shard_num = false; + /// Results of scalar sub queries Scalars scalars; + Scalars local_scalars; explicit TreeRewriterResult( const NamesAndTypesList & source_columns_, diff --git a/src/Interpreters/addMissingDefaults.cpp b/src/Interpreters/addMissingDefaults.cpp index f6d5f83fce3..04e1e6856cc 100644 --- a/src/Interpreters/addMissingDefaults.cpp +++ b/src/Interpreters/addMissingDefaults.cpp @@ -83,10 +83,10 @@ ActionsDAGPtr addMissingDefaults( /// Computes explicitly specified values by default and materialized columns. if (auto dag = evaluateMissingDefaults(actions->getResultColumns(), required_columns, columns, context, true, null_as_default)) actions = ActionsDAG::merge(std::move(*actions), std::move(*dag)); - else - /// Removes unused columns and reorders result. - /// The same is done in evaluateMissingDefaults if not empty dag is returned. - actions->removeUnusedActions(required_columns.getNames()); + + /// Removes unused columns and reorders result. + actions->removeUnusedActions(required_columns.getNames(), false); + actions->addMaterializingOutputActions(); return actions; } diff --git a/src/Interpreters/executeDDLQueryOnCluster.cpp b/src/Interpreters/executeDDLQueryOnCluster.cpp index d3a9bcbe1e4..ce00676b2ed 100644 --- a/src/Interpreters/executeDDLQueryOnCluster.cpp +++ b/src/Interpreters/executeDDLQueryOnCluster.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace fs = std::filesystem; @@ -124,7 +125,7 @@ BlockIO executeDDLQueryOnCluster(const ASTPtr & query_ptr_, ContextPtr context, use_local_default_database = true; } } - std::sort(shard_default_databases.begin(), shard_default_databases.end()); + ::sort(shard_default_databases.begin(), shard_default_databases.end()); shard_default_databases.erase(std::unique(shard_default_databases.begin(), shard_default_databases.end()), shard_default_databases.end()); assert(use_local_default_database || !shard_default_databases.empty()); @@ -324,7 +325,7 @@ Chunk DDLQueryStatusSource::generate() return {}; } - LOG_INFO(log, msg_format, node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts); + LOG_INFO(log, fmt::runtime(msg_format), node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts); NameSet unfinished_hosts = waiting_hosts; for (const auto & host_id : finished_hosts) diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp index 870e01d3b5c..f40d35e970b 100644 --- a/src/Interpreters/executeQuery.cpp +++ b/src/Interpreters/executeQuery.cpp @@ -61,6 +61,7 @@ #include #include +#include #include @@ -659,7 +660,7 @@ static std::tuple executeQueryImpl( if (context->query_trace_context.trace_id != UUID()) { auto * raw_interpreter_ptr = interpreter.get(); - std::string class_name(abi::__cxa_demangle(typeid(*raw_interpreter_ptr).name(), nullptr, nullptr, nullptr)); + std::string class_name(demangle(typeid(*raw_interpreter_ptr).name())); span = std::make_unique(class_name + "::execute()"); } res = interpreter->execute(); diff --git a/src/Interpreters/inplaceBlockConversions.cpp b/src/Interpreters/inplaceBlockConversions.cpp index 2841abe757e..5c87950b821 100644 --- a/src/Interpreters/inplaceBlockConversions.cpp +++ b/src/Interpreters/inplaceBlockConversions.cpp @@ -134,7 +134,6 @@ ActionsDAGPtr createExpressions( const Block & header, ASTPtr expr_list, bool save_unneeded_columns, - const NamesAndTypesList & required_columns, ContextPtr context) { if (!expr_list) @@ -146,12 +145,6 @@ ActionsDAGPtr createExpressions( auto actions = expression_analyzer.getActionsDAG(true, !save_unneeded_columns); dag = ActionsDAG::merge(std::move(*dag), std::move(*actions)); - if (save_unneeded_columns) - { - dag->removeUnusedActions(required_columns.getNames()); - dag->addMaterializingOutputActions(); - } - return dag; } @@ -163,7 +156,7 @@ void performRequiredConversions(Block & block, const NamesAndTypesList & require if (conversion_expr_list->children.empty()) return; - if (auto dag = createExpressions(block, conversion_expr_list, true, required_columns, context)) + if (auto dag = createExpressions(block, conversion_expr_list, true, context)) { auto expression = std::make_shared(std::move(dag), ExpressionActionsSettings::fromContext(context)); expression->execute(block); @@ -182,7 +175,7 @@ ActionsDAGPtr evaluateMissingDefaults( return nullptr; ASTPtr expr_list = defaultRequiredExpressions(header, required_columns, columns, null_as_default); - return createExpressions(header, expr_list, save_unneeded_columns, required_columns, context); + return createExpressions(header, expr_list, save_unneeded_columns, context); } } diff --git a/src/Interpreters/sortBlock.cpp b/src/Interpreters/sortBlock.cpp index edf911fa61c..c8a2d0903f2 100644 --- a/src/Interpreters/sortBlock.cpp +++ b/src/Interpreters/sortBlock.cpp @@ -1,13 +1,10 @@ #include -#include #include #include -#include -#include +#include #include -#include namespace DB { @@ -17,66 +14,34 @@ namespace ErrorCodes extern const int BAD_COLLATION; } -static bool isCollationRequired(const SortColumnDescription & description) +/// Column with description for sort +struct ColumnWithSortDescription +{ + const IColumn * column = nullptr; + SortColumnDescription description; + + /// It means, that this column is ColumnConst + bool column_const = false; +}; + +using ColumnsWithSortDescriptions = std::vector; + +namespace +{ + +inline bool isCollationRequired(const SortColumnDescription & description) { return description.collator != nullptr; } - -ColumnsWithSortDescriptions getColumnsWithSortDescription(const Block & block, const SortDescription & description) -{ - size_t size = description.size(); - ColumnsWithSortDescriptions res; - res.reserve(size); - - for (size_t i = 0; i < size; ++i) - { - const IColumn * column = !description[i].column_name.empty() - ? block.getByName(description[i].column_name).column.get() - : block.safeGetByPosition(description[i].column_number).column.get(); - - res.emplace_back(ColumnWithSortDescription{column, description[i], isColumnConst(*column)}); - } - - return res; -} - - -struct PartialSortingLess +template +struct PartialSortingLessImpl { const ColumnsWithSortDescriptions & columns; - explicit PartialSortingLess(const ColumnsWithSortDescriptions & columns_) : columns(columns_) {} + explicit PartialSortingLessImpl(const ColumnsWithSortDescriptions & columns_) : columns(columns_) { } - bool operator() (size_t a, size_t b) const - { - for (const auto & elem : columns) - { - int res; - if (elem.column_const) - res = 0; - else - res = elem.description.direction * elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction); - if (res < 0) - return true; - else if (res > 0) - return false; - } - return false; - } -}; - - -struct PartialSortingLessWithCollation -{ - const ColumnsWithSortDescriptions & columns; - - explicit PartialSortingLessWithCollation(const ColumnsWithSortDescriptions & columns_) - : columns(columns_) - { - } - - bool operator() (size_t a, size_t b) const + inline bool operator()(size_t a, size_t b) const { for (const auto & elem : columns) { @@ -85,13 +50,25 @@ struct PartialSortingLessWithCollation if (elem.column_const) { res = 0; + continue; } - else if (isCollationRequired(elem.description)) + + if constexpr (check_collation) { - res = elem.column->compareAtWithCollation(a, b, *elem.column, elem.description.nulls_direction, *elem.description.collator); + if (isCollationRequired(elem.description)) + { + res = elem.column->compareAtWithCollation(a, b, *elem.column, elem.description.nulls_direction, *elem.description.collator); + } + else + { + res = elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction); + } } else + { res = elem.column->compareAt(a, b, *elem.column, elem.description.nulls_direction); + } + res *= elem.description.direction; if (res < 0) return true; @@ -102,124 +79,148 @@ struct PartialSortingLessWithCollation } }; +using PartialSortingLess = PartialSortingLessImpl; +using PartialSortingLessWithCollation = PartialSortingLessImpl; + +} + +void convertTupleColumnIntoSortDescriptions( + const ColumnTuple * tuple, const SortColumnDescription & description, ColumnsWithSortDescriptions & result) +{ + for (const auto & column : tuple->getColumns()) + { + if (const auto * subtuple = typeid_cast(column.get())) + { + convertTupleColumnIntoSortDescriptions(subtuple, description, result); + } + else + { + result.emplace_back(ColumnWithSortDescription{column.get(), description, isColumnConst(*column)}); + + if (isCollationRequired(description) && !result.back().column->isCollationSupported()) + result.back().description.collator = nullptr; + } + } +} + +ColumnsWithSortDescriptions getColumnsWithSortDescription(const Block & block, const SortDescription & description) +{ + size_t size = description.size(); + + ColumnsWithSortDescriptions result; + result.reserve(size); + + for (size_t i = 0; i < size; ++i) + { + const auto & sort_column_description = description[i]; + + const IColumn * column = !sort_column_description.column_name.empty() + ? block.getByName(sort_column_description.column_name).column.get() + : block.safeGetByPosition(sort_column_description.column_number).column.get(); + + if (isCollationRequired(sort_column_description)) + { + if (!column->isCollationSupported()) + throw Exception( + "Collations could be specified only for String, LowCardinality(String), Nullable(String) or for Array or Tuple, " + "containing them.", + ErrorCodes::BAD_COLLATION); + } + + if (const auto * tuple = typeid_cast(column)) + convertTupleColumnIntoSortDescriptions(tuple, sort_column_description, result); + else + result.emplace_back(ColumnWithSortDescription{column, sort_column_description, isColumnConst(*column)}); + } + + return result; +} + void sortBlock(Block & block, const SortDescription & description, UInt64 limit) { if (!block) return; - /// If only one column to sort by - if (description.size() == 1) + ColumnsWithSortDescriptions columns_with_sort_descriptions = getColumnsWithSortDescription(block, description); + + bool all_const = true; + for (const auto & column : columns_with_sort_descriptions) { - IColumn::Permutation perm; - bool reverse = description[0].direction == -1; - - const IColumn * column = !description[0].column_name.empty() - ? block.getByName(description[0].column_name).column.get() - : block.safeGetByPosition(description[0].column_number).column.get(); - - bool is_column_const = false; - if (isCollationRequired(description[0])) + if (!column.column_const) { - if (!column->isCollationSupported()) - throw Exception("Collations could be specified only for String, LowCardinality(String), Nullable(String) or for Array or Tuple, containing them.", ErrorCodes::BAD_COLLATION); + all_const = false; + break; + } + } + if (all_const) + return; - if (isColumnConst(*column)) - is_column_const = true; - else - column->getPermutationWithCollation(*description[0].collator, reverse, limit, description[0].nulls_direction, perm); - } - else if (!isColumnConst(*column)) - { - int nan_direction_hint = description[0].nulls_direction; - column->getPermutation(reverse, limit, nan_direction_hint, perm); - } + IColumn::Permutation permutation; + + /// If only one column to sort by + if (columns_with_sort_descriptions.size() == 1) + { + auto & column_with_sort_description = columns_with_sort_descriptions[0]; + + bool reverse = column_with_sort_description.description.direction == -1; + int nan_direction_hint = column_with_sort_description.description.nulls_direction; + const auto & column = column_with_sort_description.column; + + if (isCollationRequired(column_with_sort_description.description)) + column->getPermutationWithCollation( + *column_with_sort_description.description.collator, reverse, limit, nan_direction_hint, permutation); else - /// we don't need to do anything with const column - is_column_const = true; - - size_t columns = block.columns(); - for (size_t i = 0; i < columns; ++i) - { - if (!is_column_const) - block.getByPosition(i).column = block.getByPosition(i).column->permute(perm, limit); - } + column->getPermutation(reverse, limit, nan_direction_hint, permutation); } else { size_t size = block.rows(); - IColumn::Permutation perm(size); + permutation.resize(size); for (size_t i = 0; i < size; ++i) - perm[i] = i; + permutation[i] = i; if (limit >= size) limit = 0; - bool need_collation = false; - ColumnsWithSortDescriptions columns_with_sort_desc = getColumnsWithSortDescription(block, description); + EqualRanges ranges; + ranges.emplace_back(0, permutation.size()); - for (size_t i = 0, num_sort_columns = description.size(); i < num_sort_columns; ++i) + for (const auto & column_with_sort_description : columns_with_sort_descriptions) { - const IColumn * column = columns_with_sort_desc[i].column; - if (isCollationRequired(description[i])) - { - if (!column->isCollationSupported()) - throw Exception("Collations could be specified only for String, LowCardinality(String), Nullable(String) or for Array or Tuple, containing them.", ErrorCodes::BAD_COLLATION); + while (!ranges.empty() && limit && limit <= ranges.back().first) + ranges.pop_back(); - need_collation = true; + if (ranges.empty()) + break; + + if (column_with_sort_description.column_const) + continue; + + bool is_collation_required = isCollationRequired(column_with_sort_description.description); + bool reverse = column_with_sort_description.description.direction < 0; + int nan_direction_hint = column_with_sort_description.description.nulls_direction; + const auto & column = column_with_sort_description.column; + + if (is_collation_required) + { + column->updatePermutationWithCollation( + *column_with_sort_description.description.collator, reverse, limit, nan_direction_hint, permutation, ranges); + } + else + { + column->updatePermutation(reverse, limit, nan_direction_hint, permutation, ranges); } } + } - if (need_collation) - { - EqualRanges ranges; - ranges.emplace_back(0, perm.size()); - for (const auto & column : columns_with_sort_desc) - { - while (!ranges.empty() && limit && limit <= ranges.back().first) - ranges.pop_back(); - - if (ranges.empty()) - break; - - if (column.column_const) - continue; - - if (isCollationRequired(column.description)) - { - column.column->updatePermutationWithCollation( - *column.description.collator, column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges); - } - else - { - column.column->updatePermutation( - column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges); - } - } - } - else - { - EqualRanges ranges; - ranges.emplace_back(0, perm.size()); - for (const auto & column : columns_with_sort_desc) - { - while (!ranges.empty() && limit && limit <= ranges.back().first) - ranges.pop_back(); - - if (ranges.empty()) - break; - - column.column->updatePermutation( - column.description.direction < 0, limit, column.description.nulls_direction, perm, ranges); - } - } - - size_t columns = block.columns(); - for (size_t i = 0; i < columns; ++i) - block.getByPosition(i).column = block.getByPosition(i).column->permute(perm, limit); + size_t columns = block.columns(); + for (size_t i = 0; i < columns; ++i) + { + auto & column_to_sort = block.getByPosition(i).column; + column_to_sort = column_to_sort->permute(permutation, limit); } } - void stableGetPermutation(const Block & block, const SortDescription & description, IColumn::Permutation & out_permutation) { if (!block) @@ -235,7 +236,6 @@ void stableGetPermutation(const Block & block, const SortDescription & descripti std::stable_sort(out_permutation.begin(), out_permutation.end(), PartialSortingLess(columns_with_sort_desc)); } - bool isAlreadySorted(const Block & block, const SortDescription & description) { if (!block) @@ -276,12 +276,15 @@ void stableSortBlock(Block & block, const SortDescription & description) if (!block) return; - IColumn::Permutation perm; - stableGetPermutation(block, description, perm); + IColumn::Permutation permutation; + stableGetPermutation(block, description, permutation); size_t columns = block.columns(); for (size_t i = 0; i < columns; ++i) - block.safeGetByPosition(i).column = block.safeGetByPosition(i).column->permute(perm, 0); + { + auto & column_to_sort = block.safeGetByPosition(i).column; + column_to_sort = column_to_sort->permute(permutation, 0); + } } } diff --git a/src/Interpreters/sortBlock.h b/src/Interpreters/sortBlock.h index faf9384901b..31ae78e90b0 100644 --- a/src/Interpreters/sortBlock.h +++ b/src/Interpreters/sortBlock.h @@ -10,7 +10,6 @@ namespace DB /// Sort one block by `description`. If limit != 0, then the partial sort of the first `limit` rows is produced. void sortBlock(Block & block, const SortDescription & description, UInt64 limit = 0); - /** Used only in StorageMergeTree to sort the data with INSERT. * Sorting is stable. This is important for keeping the order of rows in the CollapsingMergeTree engine * - because based on the order of rows it is determined whether to delete or leave groups of rows when collapsing. @@ -23,24 +22,9 @@ void stableSortBlock(Block & block, const SortDescription & description); */ void stableGetPermutation(const Block & block, const SortDescription & description, IColumn::Permutation & out_permutation); - /** Quickly check whether the block is already sorted. If the block is not sorted - returns false as fast as possible. * Collations are not supported. */ bool isAlreadySorted(const Block & block, const SortDescription & description); -/// Column with description for sort -struct ColumnWithSortDescription -{ - const IColumn * column = nullptr; - SortColumnDescription description; - - /// It means, that this column is ColumnConst - bool column_const = false; -}; - -using ColumnsWithSortDescriptions = std::vector; - -ColumnsWithSortDescriptions getColumnsWithSortDescription(const Block & block, const SortDescription & description); - } diff --git a/src/Parsers/ASTCreateQuery.cpp b/src/Parsers/ASTCreateQuery.cpp index e61a0f55142..23881cd3fbb 100644 --- a/src/Parsers/ASTCreateQuery.cpp +++ b/src/Parsers/ASTCreateQuery.cpp @@ -198,8 +198,6 @@ ASTPtr ASTCreateQuery::clone() const res->set(res->storage, storage->clone()); if (select) res->set(res->select, select->clone()); - if (tables) - res->set(res->tables, tables->clone()); if (table_overrides) res->set(res->table_overrides, table_overrides->clone()); @@ -434,12 +432,6 @@ void ASTCreateQuery::formatQueryImpl(const FormatSettings & settings, FormatStat settings.ostr << (comment ? ")" : ""); } - if (tables) - { - settings.ostr << (settings.hilite ? hilite_keyword : "") << " WITH " << (settings.hilite ? hilite_none : ""); - tables->formatImpl(settings, state, frame); - } - if (comment) { settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws << "COMMENT " << (settings.hilite ? hilite_none : ""); diff --git a/src/Parsers/ASTCreateQuery.h b/src/Parsers/ASTCreateQuery.h index fcc4107bb5f..a0070892b79 100644 --- a/src/Parsers/ASTCreateQuery.h +++ b/src/Parsers/ASTCreateQuery.h @@ -73,7 +73,6 @@ public: bool replace_view{false}; /// CREATE OR REPLACE VIEW ASTColumns * columns_list = nullptr; - ASTExpressionList * tables = nullptr; StorageID to_table_id = StorageID::createEmpty(); /// For CREATE MATERIALIZED VIEW mv TO table. UUID to_inner_uuid = UUIDHelpers::Nil; /// For materialized view with inner table diff --git a/src/Parsers/ASTFunction.cpp b/src/Parsers/ASTFunction.cpp index 36731a3acd0..f9a5c7be75f 100644 --- a/src/Parsers/ASTFunction.cpp +++ b/src/Parsers/ASTFunction.cpp @@ -32,6 +32,15 @@ void ASTFunction::appendColumnNameImpl(WriteBuffer & ostr) const if (name == "view") throw Exception("Table function view cannot be used as an expression", ErrorCodes::UNEXPECTED_EXPRESSION); + /// If function can be converted to literal it will be parsed as literal after formatting. + /// In distributed query it may lead to mismathed column names. + /// To avoid it we check whether we can convert function to literal. + if (auto literal = toLiteral()) + { + literal->appendColumnName(ostr); + return; + } + writeString(name, ostr); if (parameters) @@ -111,31 +120,42 @@ void ASTFunction::updateTreeHashImpl(SipHash & hash_state) const IAST::updateTreeHashImpl(hash_state); } +template +static ASTPtr createLiteral(const ASTs & arguments) +{ + Container container; + + for (const auto & arg : arguments) + { + if (const auto * literal = arg->as()) + { + container.push_back(literal->value); + } + else if (auto * func = arg->as()) + { + if (auto func_literal = func->toLiteral()) + container.push_back(func_literal->as()->value); + else + return {}; + } + else + /// Some of the Array or Tuple arguments is not literal + return {}; + } + + return std::make_shared(container); +} ASTPtr ASTFunction::toLiteral() const { - if (!arguments) return {}; + if (!arguments) + return {}; if (name == "array") - { - Array array; + return createLiteral(arguments->children); - for (const auto & arg : arguments->children) - { - if (auto * literal = arg->as()) - array.push_back(literal->value); - else if (auto * func = arg->as()) - { - if (auto func_literal = func->toLiteral()) - array.push_back(func_literal->as()->value); - } - else - /// Some of the Array arguments is not literal - return {}; - } - - return std::make_shared(array); - } + if (name == "tuple") + return createLiteral(arguments->children); return {}; } diff --git a/src/Parsers/ASTProjectionDeclaration.cpp b/src/Parsers/ASTProjectionDeclaration.cpp index 740a2fe3efd..60050986161 100644 --- a/src/Parsers/ASTProjectionDeclaration.cpp +++ b/src/Parsers/ASTProjectionDeclaration.cpp @@ -20,7 +20,7 @@ void ASTProjectionDeclaration::formatImpl(const FormatSettings & settings, Forma settings.ostr << backQuoteIfNeed(name); std::string indent_str = settings.one_line ? "" : std::string(4u * frame.indent, ' '); std::string nl_or_nothing = settings.one_line ? "" : "\n"; - settings.ostr << nl_or_nothing << indent_str << "(" << nl_or_nothing; + settings.ostr << settings.nl_or_ws << indent_str << "(" << nl_or_nothing; FormatStateStacked frame_nested = frame; frame_nested.need_parens = false; ++frame_nested.indent; diff --git a/src/Parsers/ASTProjectionSelectQuery.cpp b/src/Parsers/ASTProjectionSelectQuery.cpp index 7a855eb2be2..8526c7aef26 100644 --- a/src/Parsers/ASTProjectionSelectQuery.cpp +++ b/src/Parsers/ASTProjectionSelectQuery.cpp @@ -72,8 +72,18 @@ void ASTProjectionSelectQuery::formatImpl(const FormatSettings & s, FormatState if (orderBy()) { + /// Let's convert the ASTFunction into ASTExpressionList, which generates consistent format + /// between GROUP BY and ORDER BY projection definition. s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "ORDER BY " << (s.hilite ? hilite_none : ""); - orderBy()->formatImpl(s, state, frame); + ASTPtr order_by; + if (auto * func = orderBy()->as()) + order_by = func->arguments; + else + { + order_by = std::make_shared(); + order_by->children.push_back(orderBy()); + } + s.one_line ? order_by->formatImpl(s, state, frame) : order_by->as().formatImplMultiline(s, state, frame); } } diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 1c5a4310f1b..3f40167b1d1 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -22,38 +22,20 @@ namespace ErrorCodes ASTPtr ASTSelectQuery::clone() const { auto res = std::make_shared(*this); + + /** NOTE Members must clone exactly in the same order in which they were inserted into `children` in ParserSelectQuery. + * This is important because the AST hash depends on the children order and this hash is used for multiple things, + * like the column identifiers in the case of subqueries in the IN statement or caching scalar queries (reused in CTEs so it's + * important for them to have the same hash). + * For distributed query processing, in case one of the servers is localhost and the other one is not, localhost query is executed + * within the process and is cloned, and the request is sent to the remote server in text form via TCP. + * And if the cloning order does not match the parsing order then different servers will get different identifiers. + * + * Since the positions map uses we can copy it as is and ensure the new children array is created / pushed + * in the same order as the existing one */ res->children.clear(); - res->positions.clear(); - -#define CLONE(expr) res->setExpression(expr, getExpression(expr, true)) - - /** NOTE Members must clone exactly in the same order, - * in which they were inserted into `children` in ParserSelectQuery. - * This is important because of the children's names the identifier (getTreeHash) is compiled, - * which can be used for column identifiers in the case of subqueries in the IN statement. - * For distributed query processing, in case one of the servers is localhost and the other one is not, - * localhost query is executed within the process and is cloned, - * and the request is sent to the remote server in text form via TCP. - * And if the cloning order does not match the parsing order, - * then different servers will get different identifiers. - */ - CLONE(Expression::WITH); - CLONE(Expression::SELECT); - CLONE(Expression::TABLES); - CLONE(Expression::PREWHERE); - CLONE(Expression::WHERE); - CLONE(Expression::GROUP_BY); - CLONE(Expression::HAVING); - CLONE(Expression::WINDOW); - CLONE(Expression::ORDER_BY); - CLONE(Expression::LIMIT_BY_OFFSET); - CLONE(Expression::LIMIT_BY_LENGTH); - CLONE(Expression::LIMIT_BY); - CLONE(Expression::LIMIT_OFFSET); - CLONE(Expression::LIMIT_LENGTH); - CLONE(Expression::SETTINGS); - -#undef CLONE + for (const auto & child : children) + res->children.push_back(child->clone()); return res; } diff --git a/src/Parsers/ASTTTLElement.cpp b/src/Parsers/ASTTTLElement.cpp index 2d22c1b4307..90278e27c0c 100644 --- a/src/Parsers/ASTTTLElement.cpp +++ b/src/Parsers/ASTTTLElement.cpp @@ -1,13 +1,17 @@ - #include #include #include #include - +#include namespace DB { +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} + ASTPtr ASTTTLElement::clone() const { auto clone = std::make_shared(*this); @@ -29,13 +33,21 @@ ASTPtr ASTTTLElement::clone() const void ASTTTLElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { ttl()->formatImpl(settings, state, frame); - if (mode == TTLMode::MOVE && destination_type == DataDestinationType::DISK) + if (mode == TTLMode::MOVE) { - settings.ostr << " TO DISK " << quoteString(destination_name); - } - else if (mode == TTLMode::MOVE && destination_type == DataDestinationType::VOLUME) - { - settings.ostr << " TO VOLUME " << quoteString(destination_name); + if (destination_type == DataDestinationType::DISK) + settings.ostr << " TO DISK "; + else if (destination_type == DataDestinationType::VOLUME) + settings.ostr << " TO VOLUME "; + else + throw Exception(ErrorCodes::LOGICAL_ERROR, + "Unsupported destination type {} for TTL MOVE", + magic_enum::enum_name(destination_type)); + + if (if_exists) + settings.ostr << "IF EXISTS "; + + settings.ostr << quoteString(destination_name); } else if (mode == TTLMode::GROUP_BY) { diff --git a/src/Parsers/ASTTTLElement.h b/src/Parsers/ASTTTLElement.h index ce011d76c7b..a396a4c54e0 100644 --- a/src/Parsers/ASTTTLElement.h +++ b/src/Parsers/ASTTTLElement.h @@ -16,16 +16,18 @@ public: TTLMode mode; DataDestinationType destination_type; String destination_name; + bool if_exists = false; ASTs group_by_key; ASTs group_by_assignments; ASTPtr recompression_codec; - ASTTTLElement(TTLMode mode_, DataDestinationType destination_type_, const String & destination_name_) + ASTTTLElement(TTLMode mode_, DataDestinationType destination_type_, const String & destination_name_, bool if_exists_) : mode(mode_) , destination_type(destination_type_) , destination_name(destination_name_) + , if_exists(if_exists_) , ttl_expr_pos(-1) , where_expr_pos(-1) { diff --git a/src/Parsers/ASTTablesInSelectQuery.cpp b/src/Parsers/ASTTablesInSelectQuery.cpp index 4680acc4c64..7435b22f7b7 100644 --- a/src/Parsers/ASTTablesInSelectQuery.cpp +++ b/src/Parsers/ASTTablesInSelectQuery.cpp @@ -247,10 +247,12 @@ void ASTTableJoin::formatImpl(const FormatSettings & settings, FormatState & sta void ASTArrayJoin::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const { + std::string indent_str = settings.one_line ? "" : std::string(4 * frame.indent, ' '); frame.expression_list_prepend_whitespace = true; settings.ostr << (settings.hilite ? hilite_keyword : "") << settings.nl_or_ws + << indent_str << (kind == Kind::Left ? "LEFT " : "") << "ARRAY JOIN" << (settings.hilite ? hilite_none : ""); settings.one_line diff --git a/src/Parsers/DumpASTNode.h b/src/Parsers/DumpASTNode.h index e1071f02dd3..e8efeb4b59c 100644 --- a/src/Parsers/DumpASTNode.h +++ b/src/Parsers/DumpASTNode.h @@ -102,7 +102,7 @@ public: ~DebugASTLog() { if constexpr (_enable) - LOG_DEBUG(log, buf.str()); + LOG_DEBUG(log, fmt::runtime(buf.str())); } WriteBuffer * stream() { return (_enable ? &buf : nullptr); } diff --git a/src/Parsers/ExpressionElementParsers.cpp b/src/Parsers/ExpressionElementParsers.cpp index 9c8f8b4e46b..e00e0aba7b3 100644 --- a/src/Parsers/ExpressionElementParsers.cpp +++ b/src/Parsers/ExpressionElementParsers.cpp @@ -2360,6 +2360,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ParserKeyword s_to_disk("TO DISK"); ParserKeyword s_to_volume("TO VOLUME"); + ParserKeyword s_if_exists("IF EXISTS"); ParserKeyword s_delete("DELETE"); ParserKeyword s_where("WHERE"); ParserKeyword s_group_by("GROUP BY"); @@ -2414,9 +2415,13 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr group_by_key; ASTPtr recompression_codec; ASTPtr group_by_assignments; + bool if_exists = false; if (mode == TTLMode::MOVE) { + if (s_if_exists.ignore(pos)) + if_exists = true; + ASTPtr ast_space_name; if (!parser_string_literal.parse(pos, ast_space_name, expected)) return false; @@ -2448,7 +2453,7 @@ bool ParserTTLElement::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; } - auto ttl_element = std::make_shared(mode, destination_type, destination_name); + auto ttl_element = std::make_shared(mode, destination_type, destination_name, if_exists); ttl_element->setTTL(std::move(ttl_expr)); if (where_expr) ttl_element->setWhere(std::move(where_expr)); diff --git a/src/Parsers/ExpressionListParsers.cpp b/src/Parsers/ExpressionListParsers.cpp index 96c1bad75c2..13af308736b 100644 --- a/src/Parsers/ExpressionListParsers.cpp +++ b/src/Parsers/ExpressionListParsers.cpp @@ -689,7 +689,7 @@ bool ParserUnaryExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expec bool ParserCastExpression::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { ASTPtr expr_ast; - if (!ParserExpressionElement().parse(pos, expr_ast, expected)) + if (!elem_parser->parse(pos, expr_ast, expected)) return false; ASTPtr type_ast; @@ -711,7 +711,7 @@ bool ParserArrayElementExpression::parseImpl(Pos & pos, ASTPtr & node, Expected { return ParserLeftAssociativeBinaryOperatorList{ operators, - std::make_unique(), + std::make_unique(std::make_unique()), std::make_unique(false) }.parse(pos, node, expected); } @@ -721,7 +721,7 @@ bool ParserTupleElementExpression::parseImpl(Pos & pos, ASTPtr & node, Expected { return ParserLeftAssociativeBinaryOperatorList{ operators, - std::make_unique(), + std::make_unique(std::make_unique()), std::make_unique() }.parse(pos, node, expected); } diff --git a/src/Parsers/ExpressionListParsers.h b/src/Parsers/ExpressionListParsers.h index a035d4a2ef0..358fe778f91 100644 --- a/src/Parsers/ExpressionListParsers.h +++ b/src/Parsers/ExpressionListParsers.h @@ -203,6 +203,15 @@ protected: /// Example: "[1, 1 + 1, 1 + 2]::Array(UInt8)" class ParserCastExpression : public IParserBase { +private: + ParserPtr elem_parser; + +public: + ParserCastExpression(ParserPtr && elem_parser_) + : elem_parser(std::move(elem_parser_)) + { + } + protected: const char * getName() const override { return "CAST expression"; } @@ -238,7 +247,7 @@ class ParserUnaryExpression : public IParserBase { private: static const char * operators[]; - ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique()}; + ParserPrefixUnaryOperatorExpression operator_parser {operators, std::make_unique(std::make_unique())}; protected: const char * getName() const override { return "unary expression"; } diff --git a/src/Parsers/Lexer.cpp b/src/Parsers/Lexer.cpp index 3ac9d417b46..fd5d0b556f0 100644 --- a/src/Parsers/Lexer.cpp +++ b/src/Parsers/Lexer.cpp @@ -280,6 +280,18 @@ Token Lexer::nextTokenImpl() } return Token(TokenType::Slash, token_begin, pos); } + case '#': /// start of single line comment, MySQL style + { /// PostgreSQL has some operators using '#' character. + /// For less ambiguity, we will recognize a comment only if # is followed by whitespace. + /// or #! as a special case for "shebang". + /// #hello - not a comment + /// # hello - a comment + /// #!/usr/bin/clickhouse-local --queries-file - a comment + ++pos; + if (pos < end && (*pos == ' ' || *pos == '!')) + return comment_until_end_of_line(); + return Token(TokenType::Error, token_begin, pos); + } case '%': return Token(TokenType::Percent, token_begin, ++pos); case '=': /// =, == @@ -335,6 +347,13 @@ Token Lexer::nextTokenImpl() return Token(TokenType::DoubleAt, token_begin, ++pos); return Token(TokenType::At, token_begin, pos); } + case '\\': + { + ++pos; + if (pos < end && *pos == 'G') + return Token(TokenType::VerticalDelimiter, token_begin, ++pos); + return Token(TokenType::Error, token_begin, pos); + } default: if (*pos == '$') diff --git a/src/Parsers/Lexer.h b/src/Parsers/Lexer.h index f41e05147e5..ec472fb1a36 100644 --- a/src/Parsers/Lexer.h +++ b/src/Parsers/Lexer.h @@ -28,6 +28,7 @@ namespace DB \ M(Comma) \ M(Semicolon) \ + M(VerticalDelimiter) /** Vertical delimiter \G */ \ M(Dot) /** Compound identifiers, like a.b or tuple access operator a.1, (x, y).2. */ \ /** Need to be distinguished from floating point number with omitted integer part: .1 */ \ \ diff --git a/src/Parsers/ParserCreateQuery.cpp b/src/Parsers/ParserCreateQuery.cpp index 6d295a0d516..9c9989dc39f 100644 --- a/src/Parsers/ParserCreateQuery.cpp +++ b/src/Parsers/ParserCreateQuery.cpp @@ -18,6 +18,7 @@ #include #include #include +#include namespace DB @@ -353,20 +354,26 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr ttl_table; ASTPtr settings; - if (!s_engine.ignore(pos, expected)) - return false; + bool storage_like = false; - s_eq.ignore(pos, expected); + if (s_engine.ignore(pos, expected)) + { + s_eq.ignore(pos, expected); - if (!ident_with_optional_params_p.parse(pos, engine, expected)) - return false; + if (!ident_with_optional_params_p.parse(pos, engine, expected)) + return false; + storage_like = true; + } while (true) { if (!partition_by && s_partition_by.ignore(pos, expected)) { if (expression_p.parse(pos, partition_by, expected)) + { + storage_like = true; continue; + } else return false; } @@ -374,7 +381,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!primary_key && s_primary_key.ignore(pos, expected)) { if (expression_p.parse(pos, primary_key, expected)) + { + storage_like = true; continue; + } else return false; } @@ -382,7 +392,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!order_by && s_order_by.ignore(pos, expected)) { if (expression_p.parse(pos, order_by, expected)) + { + storage_like = true; continue; + } else return false; } @@ -390,7 +403,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!sample_by && s_sample_by.ignore(pos, expected)) { if (expression_p.parse(pos, sample_by, expected)) + { + storage_like = true; continue; + } else return false; } @@ -398,7 +414,10 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) if (!ttl_table && s_ttl.ignore(pos, expected)) { if (parser_ttl_list.parse(pos, ttl_table, expected)) + { + storage_like = true; continue; + } else return false; } @@ -407,10 +426,14 @@ bool ParserStorage::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) { if (!settings_p.parse(pos, settings, expected)) return false; + storage_like = true; } break; } + // If any part of storage definition is found create storage node + if (!storage_like) + return false; auto storage = std::make_shared(); storage->set(storage->engine, engine); @@ -549,13 +572,11 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (!storage_parse_result && !is_temporary) { - if (!s_as.ignore(pos, expected)) + if (s_as.ignore(pos, expected) && !table_function_p.parse(pos, as_table_function, expected)) return false; - if (!table_function_p.parse(pos, as_table_function, expected)) - { - return false; - } } + + /// Will set default table engine if Storage clause was not parsed } /** Create queries without list of columns: * - CREATE|ATTACH TABLE ... AS ... @@ -590,10 +611,6 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe } } } - else if (!storage) - { - return false; - } } auto comment = parseComment(pos, expected); @@ -625,12 +642,14 @@ bool ParserCreateTableQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expe if (comment) query->set(query->comment, comment); - if (query->storage && query->columns_list && query->columns_list->primary_key) + if (query->columns_list && query->columns_list->primary_key) { - if (query->storage->primary_key) - { + /// If engine is not set will use default one + if (!query->storage) + query->set(query->storage, std::make_shared()); + else if (query->storage->primary_key) throw Exception("Multiple primary keys are not allowed.", ErrorCodes::BAD_ARGUMENTS); - } + query->storage->primary_key = query->columns_list->primary_key; } @@ -1263,8 +1282,8 @@ bool ParserCreateViewQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expec if (is_materialized_view && !to_table) { /// Internal ENGINE for MATERIALIZED VIEW must be specified. - if (!storage_p.parse(pos, storage, expected)) - return false; + /// Actually check it in Interpreter as default_table_engine can be set + storage_p.parse(pos, storage, expected); if (s_populate.ignore(pos, expected)) is_populate = true; diff --git a/src/Parsers/ParserCreateQuery.h b/src/Parsers/ParserCreateQuery.h index 615121eae58..c48cea9c480 100644 --- a/src/Parsers/ParserCreateQuery.h +++ b/src/Parsers/ParserCreateQuery.h @@ -276,7 +276,7 @@ protected: class ParserIndexDeclaration : public IParserBase { public: - ParserIndexDeclaration() {} + ParserIndexDeclaration() = default; protected: const char * getName() const override { return "index declaration"; } @@ -336,7 +336,7 @@ protected: /** - * ENGINE = name [PARTITION BY expr] [ORDER BY expr] [PRIMARY KEY expr] [SAMPLE BY expr] [SETTINGS name = value, ...] + * [ENGINE = name] [PARTITION BY expr] [ORDER BY expr] [PRIMARY KEY expr] [SAMPLE BY expr] [SETTINGS name = value, ...] */ class ParserStorage : public IParserBase { @@ -391,7 +391,7 @@ class ParserTableOverrideDeclaration : public IParserBase { public: const bool is_standalone; - ParserTableOverrideDeclaration(bool is_standalone_ = true) : is_standalone(is_standalone_) { } + explicit ParserTableOverrideDeclaration(bool is_standalone_ = true) : is_standalone(is_standalone_) { } protected: const char * getName() const override { return "table override declaration"; } diff --git a/src/Parsers/ParserDictionary.cpp b/src/Parsers/ParserDictionary.cpp index 399dda08911..ef914e2264a 100644 --- a/src/Parsers/ParserDictionary.cpp +++ b/src/Parsers/ParserDictionary.cpp @@ -188,8 +188,19 @@ bool ParserDictionary::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) ASTPtr ast_settings; /// Primary is required to be the first in dictionary definition - if (primary_key_keyword.ignore(pos) && !expression_list_p.parse(pos, primary_key, expected)) - return false; + if (primary_key_keyword.ignore(pos)) + { + bool was_open = false; + + if (open.ignore(pos, expected)) + was_open = true; + + if (!expression_list_p.parse(pos, primary_key, expected)) + return false; + + if (was_open && !close.ignore(pos, expected)) + return false; + } /// Loop is used to avoid strict order of dictionary properties while (true) diff --git a/src/Parsers/ParserProjectionSelectQuery.cpp b/src/Parsers/ParserProjectionSelectQuery.cpp index 0467f84de2a..b2adb5cf154 100644 --- a/src/Parsers/ParserProjectionSelectQuery.cpp +++ b/src/Parsers/ParserProjectionSelectQuery.cpp @@ -55,6 +55,7 @@ bool ParserProjectionSelectQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & return false; } + // ORDER BY needs to be an ASTFunction so that we can use it as a sorting key if (s_order_by.ignore(pos, expected)) { ASTPtr expr_list; diff --git a/src/Parsers/ParserSystemQuery.cpp b/src/Parsers/ParserSystemQuery.cpp index 5a61929bdb3..915a01cc063 100644 --- a/src/Parsers/ParserSystemQuery.cpp +++ b/src/Parsers/ParserSystemQuery.cpp @@ -48,7 +48,7 @@ static bool parseQueryWithOnClusterAndMaybeTable(std::shared_ptr parsed_table = parseDatabaseAndTableAsAST(pos, expected, res->database, res->table); if (!parsed_table && require_table) - return false; + return false; if (!parsed_on_cluster && ParserKeyword{"ON"}.ignore(pos, expected)) if (!ASTQueryWithOnCluster::parse(pos, cluster, expected)) @@ -64,6 +64,20 @@ static bool parseQueryWithOnClusterAndMaybeTable(std::shared_ptr return true; } +static bool parseQueryWithOnCluster(std::shared_ptr & res, IParser::Pos & pos, + Expected & expected) +{ + String cluster_str; + if (ParserKeyword{"ON"}.ignore(pos, expected)) + { + if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) + return false; + } + res->cluster = cluster_str; + + return true; +} + bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & expected) { if (!ParserKeyword{"SYSTEM"}.ignore(pos, expected)) @@ -98,13 +112,8 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } case Type::RELOAD_MODEL: { - String cluster_str; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) - return false; - } - res->cluster = cluster_str; + parseQueryWithOnCluster(res, pos, expected); + ASTPtr ast; if (ParserStringLiteral{}.parse(pos, ast, expected)) { @@ -127,13 +136,8 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } case Type::RELOAD_FUNCTION: { - String cluster_str; - if (ParserKeyword{"ON"}.ignore(pos, expected)) - { - if (!ASTQueryWithOnCluster::parse(pos, cluster_str, expected)) - return false; - } - res->cluster = cluster_str; + parseQueryWithOnCluster(res, pos, expected); + ASTPtr ast; if (ParserStringLiteral{}.parse(pos, ast, expected)) { @@ -156,6 +160,8 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } case Type::DROP_REPLICA: { + parseQueryWithOnCluster(res, pos, expected); + ASTPtr ast; if (!ParserStringLiteral{}.parse(pos, ast, expected)) return false; @@ -196,12 +202,17 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & case Type::RESTART_REPLICA: case Type::SYNC_REPLICA: + { + parseQueryWithOnCluster(res, pos, expected); if (!parseDatabaseAndTableAsAST(pos, expected, res->database, res->table)) return false; break; + } case Type::RESTART_DISK: { + parseQueryWithOnCluster(res, pos, expected); + ASTPtr ast; if (ParserIdentifier{}.parse(pos, ast, expected)) res->disk = ast->as().name(); @@ -235,7 +246,7 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & String storage_policy_str; String volume_str; - if (ParserKeyword{"ON VOLUME"}.ignore(pos, expected)) + auto parse_on_volume = [&]() -> bool { ASTPtr ast; if (ParserIdentifier{}.parse(pos, ast, expected)) @@ -250,7 +261,25 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & volume_str = ast->as().name(); else return false; + + return true; + }; + + if (ParserKeyword{"ON VOLUME"}.ignore(pos, expected)) + { + if (!parse_on_volume()) + return false; } + else + { + parseQueryWithOnCluster(res, pos, expected); + if (ParserKeyword{"ON VOLUME"}.ignore(pos, expected)) + { + if (!parse_on_volume()) + return false; + } + } + res->storage_policy = storage_policy_str; res->volume = volume_str; if (res->volume.empty() && res->storage_policy.empty()) @@ -268,11 +297,14 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & case Type::START_REPLICATED_SENDS: case Type::STOP_REPLICATION_QUEUES: case Type::START_REPLICATION_QUEUES: + parseQueryWithOnCluster(res, pos, expected); parseDatabaseAndTableAsAST(pos, expected, res->database, res->table); break; case Type::SUSPEND: { + parseQueryWithOnCluster(res, pos, expected); + ASTPtr seconds; if (!(ParserKeyword{"FOR"}.ignore(pos, expected) && ParserUnsignedInteger().parse(pos, seconds, expected) @@ -286,8 +318,10 @@ bool ParserSystemQuery::parseImpl(IParser::Pos & pos, ASTPtr & node, Expected & } default: - /// There are no [db.table] after COMMAND NAME + { + parseQueryWithOnCluster(res, pos, expected); break; + } } if (res->database) diff --git a/src/Processors/DelayedPortsProcessor.cpp b/src/Processors/DelayedPortsProcessor.cpp index 8174619f8ce..24023529bca 100644 --- a/src/Processors/DelayedPortsProcessor.cpp +++ b/src/Processors/DelayedPortsProcessor.cpp @@ -1,5 +1,8 @@ #include +#include + + namespace DB { @@ -18,7 +21,7 @@ InputPorts createInputPorts( return InputPorts(num_ports, header); InputPorts res; - std::sort(delayed_ports.begin(), delayed_ports.end()); + ::sort(delayed_ports.begin(), delayed_ports.end()); size_t next_delayed_port = 0; for (size_t i = 0; i < num_ports; ++i) { diff --git a/src/Processors/Executors/CompletedPipelineExecutor.cpp b/src/Processors/Executors/CompletedPipelineExecutor.cpp index 45b02cba298..8ec1916f4ce 100644 --- a/src/Processors/Executors/CompletedPipelineExecutor.cpp +++ b/src/Processors/Executors/CompletedPipelineExecutor.cpp @@ -4,7 +4,6 @@ #include #include #include -#include #include namespace DB @@ -40,11 +39,6 @@ static void threadFunction(CompletedPipelineExecutor::Data & data, ThreadGroupSt if (thread_group) CurrentThread::attachTo(thread_group); - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - data.executor->execute(num_threads); } catch (...) diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index e722f8718f7..80aacf14fe6 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -301,11 +301,6 @@ void PipelineExecutor::executeImpl(size_t num_threads) if (thread_group) CurrentThread::attachTo(thread_group); - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - try { executeSingleThread(thread_num); diff --git a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp index 0ba07df95a6..198d5ce5d8d 100644 --- a/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PullingAsyncPipelineExecutor.cpp @@ -4,9 +4,7 @@ #include #include #include - #include -#include namespace DB { @@ -77,11 +75,6 @@ static void threadFunction(PullingAsyncPipelineExecutor::Data & data, ThreadGrou if (thread_group) CurrentThread::attachTo(thread_group); - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - data.executor->execute(num_threads); } catch (...) diff --git a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp index 68898bdc2c2..6c2e62b77dc 100644 --- a/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp +++ b/src/Processors/Executors/PushingAsyncPipelineExecutor.cpp @@ -2,11 +2,8 @@ #include #include #include -#include - #include #include -#include #include namespace DB @@ -107,11 +104,6 @@ static void threadFunction(PushingAsyncPipelineExecutor::Data & data, ThreadGrou if (thread_group) CurrentThread::attachTo(thread_group); - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); - data.executor->execute(num_threads); } catch (...) diff --git a/src/Processors/ForkProcessor.cpp b/src/Processors/ForkProcessor.cpp index 9b17f8ad5ca..f4e5a5be5f2 100644 --- a/src/Processors/ForkProcessor.cpp +++ b/src/Processors/ForkProcessor.cpp @@ -63,9 +63,9 @@ ForkProcessor::Status ForkProcessor::prepare() { ++num_processed_outputs; if (num_processed_outputs == num_active_outputs) - output.push(std::move(data)); // NOLINT Can push because no full or unneeded outputs. + output.push(std::move(data)); /// NOLINT Can push because no full or unneeded outputs. else - output.push(data.clone()); + output.push(data.clone()); /// NOLINT } } diff --git a/src/Processors/Formats/IRowOutputFormat.cpp b/src/Processors/Formats/IRowOutputFormat.cpp index 33777bed519..b48c4a2b3e6 100644 --- a/src/Processors/Formats/IRowOutputFormat.cpp +++ b/src/Processors/Formats/IRowOutputFormat.cpp @@ -15,7 +15,8 @@ IRowOutputFormat::IRowOutputFormat(const Block & header, WriteBuffer & out_, con , types(header.getDataTypes()) , params(params_) { - serializations.reserve(types.size()); + num_columns = types.size(); + serializations.reserve(num_columns); for (const auto & type : types) serializations.push_back(type->getDefaultSerialization()); } @@ -68,8 +69,6 @@ void IRowOutputFormat::consumeExtremes(DB::Chunk chunk) void IRowOutputFormat::write(const Columns & columns, size_t row_num) { - size_t num_columns = columns.size(); - writeRowStartDelimiter(); for (size_t i = 0; i < num_columns; ++i) diff --git a/src/Processors/Formats/IRowOutputFormat.h b/src/Processors/Formats/IRowOutputFormat.h index 7f00313ce2d..7a57753d765 100644 --- a/src/Processors/Formats/IRowOutputFormat.h +++ b/src/Processors/Formats/IRowOutputFormat.h @@ -57,6 +57,7 @@ protected: virtual void writeAfterExtremes() {} virtual void finalizeImpl() override {} /// Write something after resultset, totals end extremes. + size_t num_columns; DataTypes types; Serializations serializations; Params params; diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.h b/src/Processors/Formats/Impl/AvroRowInputFormat.h index 46e571d87ec..1e8ee4aebb9 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.h +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.h @@ -115,7 +115,7 @@ private: std::map symbolic_skip_fn_map; }; -class AvroRowInputFormat : public IRowInputFormat +class AvroRowInputFormat final : public IRowInputFormat { public: AvroRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_); @@ -137,7 +137,7 @@ private: /// 2. SchemaRegistry: schema cache (schema_id -> schema) /// 3. AvroConfluentRowInputFormat: deserializer cache (schema_id -> AvroDeserializer) /// This is needed because KafkaStorage creates a new instance of InputFormat per a batch of messages -class AvroConfluentRowInputFormat : public IRowInputFormat +class AvroConfluentRowInputFormat final : public IRowInputFormat { public: AvroConfluentRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/AvroRowOutputFormat.h b/src/Processors/Formats/Impl/AvroRowOutputFormat.h index b07edd88ae1..0a2acc93688 100644 --- a/src/Processors/Formats/Impl/AvroRowOutputFormat.h +++ b/src/Processors/Formats/Impl/AvroRowOutputFormat.h @@ -43,7 +43,7 @@ private: std::unique_ptr traits; }; -class AvroRowOutputFormat : public IRowOutputFormat +class AvroRowOutputFormat final : public IRowOutputFormat { public: AvroRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, const FormatSettings & settings_); diff --git a/src/Processors/Formats/Impl/BinaryRowInputFormat.h b/src/Processors/Formats/Impl/BinaryRowInputFormat.h index d98e75bf621..ff7cc013cee 100644 --- a/src/Processors/Formats/Impl/BinaryRowInputFormat.h +++ b/src/Processors/Formats/Impl/BinaryRowInputFormat.h @@ -17,7 +17,7 @@ class ReadBuffer; /** A stream for inputting data in a binary line-by-line format. */ -class BinaryRowInputFormat : public RowInputFormatWithNamesAndTypes +class BinaryRowInputFormat final : public RowInputFormatWithNamesAndTypes { public: BinaryRowInputFormat(ReadBuffer & in_, Block header, Params params_, bool with_names_, bool with_types_, const FormatSettings & format_settings_); @@ -30,7 +30,7 @@ public: std::string getDiagnosticInfo() override { return {}; } }; -class BinaryFormatReader : public FormatWithNamesAndTypesReader +class BinaryFormatReader final : public FormatWithNamesAndTypesReader { public: BinaryFormatReader(ReadBuffer & in_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/BinaryRowOutputFormat.h b/src/Processors/Formats/Impl/BinaryRowOutputFormat.h index 0edfd4bfcf8..40894608677 100644 --- a/src/Processors/Formats/Impl/BinaryRowOutputFormat.h +++ b/src/Processors/Formats/Impl/BinaryRowOutputFormat.h @@ -14,7 +14,7 @@ class WriteBuffer; /** A stream for outputting data in a binary line-by-line format. */ -class BinaryRowOutputFormat: public IRowOutputFormat +class BinaryRowOutputFormat final: public IRowOutputFormat { public: BinaryRowOutputFormat(WriteBuffer & out_, const Block & header, bool with_names_, bool with_types_, const RowOutputFormatParams & params_); diff --git a/src/Processors/Formats/Impl/CSVRowOutputFormat.h b/src/Processors/Formats/Impl/CSVRowOutputFormat.h index dd9c2179f19..a36c5ff47fb 100644 --- a/src/Processors/Formats/Impl/CSVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/CSVRowOutputFormat.h @@ -14,7 +14,7 @@ class WriteBuffer; /** The stream for outputting data in csv format. * Does not conform with https://tools.ietf.org/html/rfc4180 because it uses LF, not CR LF. */ -class CSVRowOutputFormat : public IRowOutputFormat +class CSVRowOutputFormat final : public IRowOutputFormat { public: /** with_names - output in the first line a header with column names diff --git a/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h b/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h index 053de14d1a4..a8aa6ccda05 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h +++ b/src/Processors/Formats/Impl/CapnProtoRowInputFormat.h @@ -20,7 +20,7 @@ class ReadBuffer; * The schema in this case cannot be compiled in, so it uses a runtime schema parser. * See https://capnproto.org/cxx.html */ -class CapnProtoRowInputFormat : public IRowInputFormat +class CapnProtoRowInputFormat final : public IRowInputFormat { public: CapnProtoRowInputFormat(ReadBuffer & in_, Block header, Params params_, const FormatSchemaInfo & info, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.h b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.h index 6e27426f2cc..288b36508ce 100644 --- a/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.h +++ b/src/Processors/Formats/Impl/CapnProtoRowOutputFormat.h @@ -23,7 +23,7 @@ private: WriteBuffer & out; }; -class CapnProtoRowOutputFormat : public IRowOutputFormat +class CapnProtoRowOutputFormat final : public IRowOutputFormat { public: CapnProtoRowOutputFormat( diff --git a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp index c362a616647..739fa8735b2 100644 --- a/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp +++ b/src/Processors/Formats/Impl/ConstantExpressionTemplate.cpp @@ -28,6 +28,7 @@ #include #include #include +#include namespace DB @@ -299,7 +300,7 @@ ConstantExpressionTemplate::TemplateStructure::TemplateStructure(LiteralsInfo & { null_as_default = null_as_default_; - std::sort(replaced_literals.begin(), replaced_literals.end(), [](const LiteralInfo & a, const LiteralInfo & b) + ::sort(replaced_literals.begin(), replaced_literals.end(), [](const LiteralInfo & a, const LiteralInfo & b) { return a.literal->begin.value() < b.literal->begin.value(); }); diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.h b/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.h index d38d5bf0da4..a2f4509d307 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.h +++ b/src/Processors/Formats/Impl/CustomSeparatedRowInputFormat.h @@ -8,7 +8,7 @@ namespace DB { -class CustomSeparatedRowInputFormat : public RowInputFormatWithNamesAndTypes +class CustomSeparatedRowInputFormat final : public RowInputFormatWithNamesAndTypes { public: CustomSeparatedRowInputFormat( @@ -35,7 +35,7 @@ private: bool ignore_spaces; }; -class CustomSeparatedFormatReader : public FormatWithNamesAndTypesReader +class CustomSeparatedFormatReader final : public FormatWithNamesAndTypesReader { public: CustomSeparatedFormatReader(PeekableReadBuffer & buf_, bool ignore_spaces_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.h b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.h index 274df1af330..0e04764b993 100644 --- a/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.h +++ b/src/Processors/Formats/Impl/CustomSeparatedRowOutputFormat.h @@ -8,7 +8,7 @@ namespace DB class WriteBuffer; -class CustomSeparatedRowOutputFormat : public IRowOutputFormat +class CustomSeparatedRowOutputFormat final : public IRowOutputFormat { public: CustomSeparatedRowOutputFormat(const Block & header_, WriteBuffer & out_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_, bool with_names_, bool with_types_); diff --git a/src/Processors/Formats/Impl/HiveTextRowInputFormat.h b/src/Processors/Formats/Impl/HiveTextRowInputFormat.h index 3b37078acb3..8a7bee45e59 100644 --- a/src/Processors/Formats/Impl/HiveTextRowInputFormat.h +++ b/src/Processors/Formats/Impl/HiveTextRowInputFormat.h @@ -12,7 +12,7 @@ namespace DB /// A stream for input data in Hive Text format. /// Parallel parsing is disabled currently. -class HiveTextRowInputFormat : public CSVRowInputFormat +class HiveTextRowInputFormat final : public CSVRowInputFormat { public: HiveTextRowInputFormat(const Block & header_, ReadBuffer & in_, const Params & params_, const FormatSettings & format_settings_); @@ -24,7 +24,7 @@ private: const Block & header_, std::unique_ptr buf_, const Params & params_, const FormatSettings & format_settings_); }; -class HiveTextFormatReader : public CSVFormatReader +class HiveTextFormatReader final : public CSVFormatReader { public: HiveTextFormatReader(std::unique_ptr buf_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h index ea6e9a1ed2f..9979a5d1474 100644 --- a/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONAsStringRowInputFormat.h @@ -15,7 +15,7 @@ class ReadBuffer; /// Each JSON object is parsed as a whole to string. /// This format can only parse a table with single field of type String. -class JSONAsStringRowInputFormat : public IRowInputFormat +class JSONAsStringRowInputFormat final : public IRowInputFormat { public: JSONAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h index efa0604fc6c..79c76214774 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowInputFormat.h @@ -19,7 +19,7 @@ class ReadBuffer; * - JSONCompactStringsEachRowWithNamesAndTypes * */ -class JSONCompactEachRowRowInputFormat : public RowInputFormatWithNamesAndTypes +class JSONCompactEachRowRowInputFormat final : public RowInputFormatWithNamesAndTypes { public: JSONCompactEachRowRowInputFormat( @@ -38,7 +38,7 @@ private: void syncAfterError() override; }; -class JSONCompactEachRowFormatReader : public FormatWithNamesAndTypesReader +class JSONCompactEachRowFormatReader final : public FormatWithNamesAndTypesReader { public: JSONCompactEachRowFormatReader(ReadBuffer & in_, bool yield_strings_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h index 6cb78bab49d..63e9e9f1b76 100644 --- a/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactEachRowRowOutputFormat.h @@ -12,7 +12,7 @@ namespace DB /** The stream for outputting data in JSON format, by object per line. * Does not validate UTF-8. */ -class JSONCompactEachRowRowOutputFormat : public IRowOutputFormat +class JSONCompactEachRowRowOutputFormat final : public IRowOutputFormat { public: JSONCompactEachRowRowOutputFormat( diff --git a/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h b/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h index 961bd569d39..a0e9a2a6026 100644 --- a/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONCompactRowOutputFormat.h @@ -13,7 +13,7 @@ struct FormatSettings; /** The stream for outputting data in the JSONCompact- formats. */ -class JSONCompactRowOutputFormat : public JSONRowOutputFormat +class JSONCompactRowOutputFormat final : public JSONRowOutputFormat { public: JSONCompactRowOutputFormat( diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp index 6d546a3b772..549fd7a6113 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.cpp @@ -356,6 +356,7 @@ void registerInputFormatJSONEachRow(FormatFactory & factory) }); factory.registerFileExtension("ndjson", "JSONEachRow"); + factory.registerFileExtension("jsonl", "JSONEachRow"); factory.registerInputFormat("JSONStringsEachRow", []( ReadBuffer & buf, diff --git a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h index c711d3ef246..29aba696411 100644 --- a/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowRowInputFormat.h @@ -18,7 +18,7 @@ class ReadBuffer; * Fields can be listed in any order (including, in different lines there may be different order), * and some fields may be missing. */ -class JSONEachRowRowInputFormat : public IRowInputFormat +class JSONEachRowRowInputFormat final : public IRowInputFormat { public: JSONEachRowRowInputFormat( diff --git a/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.h b/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.h index fe74f7ce7a3..6bdf27a472e 100644 --- a/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.h +++ b/src/Processors/Formats/Impl/JSONEachRowWithProgressRowOutputFormat.h @@ -5,7 +5,7 @@ namespace DB { -class JSONEachRowWithProgressRowOutputFormat : public JSONEachRowRowOutputFormat +class JSONEachRowWithProgressRowOutputFormat final : public JSONEachRowRowOutputFormat { public: using JSONEachRowRowOutputFormat::JSONEachRowRowOutputFormat; diff --git a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp index 5983f3170e5..dc346b4f5f5 100644 --- a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.cpp @@ -2,6 +2,8 @@ #include #include #include +#include + namespace DB { @@ -14,7 +16,8 @@ namespace ErrorCodes LineAsStringRowInputFormat::LineAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_) : IRowInputFormat(header_, in_, std::move(params_)) { - if (header_.columns() > 1 || header_.getDataTypes()[0]->getTypeId() != TypeIndex::String) + if (header_.columns() != 1 + || !typeid_cast(header_.getByPosition(0).column.get())) { throw Exception("This input format is only suitable for tables with a single column of type String.", ErrorCodes::INCORRECT_QUERY); } @@ -27,28 +30,16 @@ void LineAsStringRowInputFormat::resetParser() void LineAsStringRowInputFormat::readLineObject(IColumn & column) { - DB::Memory<> object; + ColumnString & column_string = assert_cast(column); + auto & chars = column_string.getChars(); + auto & offsets = column_string.getOffsets(); - char * pos = in->position(); - bool need_more_data = true; + readStringUntilNewlineInto(chars, *in); + chars.push_back(0); + offsets.push_back(chars.size()); - while (loadAtPosition(*in, object, pos) && need_more_data) - { - pos = find_first_symbols<'\n'>(pos, in->buffer().end()); - if (pos == in->buffer().end()) - continue; - - if (*pos == '\n') - need_more_data = false; - - ++pos; - } - - saveUpToPosition(*in, object, pos); - loadAtPosition(*in, object, pos); - - /// Last character is always \n. - column.insertData(object.data(), object.size() - 1); + if (!in->eof()) + in->ignore(); /// Skip '\n' } bool LineAsStringRowInputFormat::readRow(MutableColumns & columns, RowReadExtension &) @@ -57,17 +48,16 @@ bool LineAsStringRowInputFormat::readRow(MutableColumns & columns, RowReadExtens return false; readLineObject(*columns[0]); - return true; } void registerInputFormatLineAsString(FormatFactory & factory) { factory.registerInputFormat("LineAsString", []( - ReadBuffer & buf, - const Block & sample, - const RowInputFormatParams & params, - const FormatSettings &) + ReadBuffer & buf, + const Block & sample, + const RowInputFormatParams & params, + const FormatSettings &) { return std::make_shared(sample, buf, params); }); @@ -76,9 +66,10 @@ void registerInputFormatLineAsString(FormatFactory & factory) void registerLineAsStringSchemaReader(FormatFactory & factory) { factory.registerExternalSchemaReader("LineAsString", []( - const FormatSettings &) + const FormatSettings &) { return std::make_shared(); }); } + } diff --git a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.h b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.h index c4c17c47dbe..080ff9985af 100644 --- a/src/Processors/Formats/Impl/LineAsStringRowInputFormat.h +++ b/src/Processors/Formats/Impl/LineAsStringRowInputFormat.h @@ -14,7 +14,7 @@ class ReadBuffer; /// Each Line object is parsed as a whole to string. /// This format can only parse a table with single field of type String. -class LineAsStringRowInputFormat : public IRowInputFormat +class LineAsStringRowInputFormat final : public IRowInputFormat { public: LineAsStringRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); diff --git a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h index 7a2aaf86f7d..c6e15282780 100644 --- a/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h +++ b/src/Processors/Formats/Impl/MarkdownRowOutputFormat.h @@ -9,7 +9,7 @@ namespace DB class ReadBuffer; -class MarkdownRowOutputFormat : public IRowOutputFormat +class MarkdownRowOutputFormat final : public IRowOutputFormat { public: MarkdownRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index 2471a98f83d..56fc5d7857b 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include @@ -12,6 +13,7 @@ #include #include #include +#include #include #include @@ -20,6 +22,8 @@ #include #include +#include + namespace DB { @@ -153,16 +157,29 @@ static void insertInteger(IColumn & column, DataTypePtr type, UInt64 value) } } -static void insertString(IColumn & column, DataTypePtr type, const char * value, size_t size) +static void insertString(IColumn & column, DataTypePtr type, const char * value, size_t size, bool bin) { auto insert_func = [&](IColumn & column_, DataTypePtr type_) { - insertString(column_, type_, value, size); + insertString(column_, type_, value, size, bin); }; if (checkAndInsertNullable(column, type, insert_func) || checkAndInsertLowCardinality(column, type, insert_func)) return; + if (isUUID(type)) + { + ReadBufferFromMemory buf(value, size); + UUID uuid; + if (bin) + readBinary(uuid, buf); + else + readUUIDText(uuid, buf); + + assert_cast(column).insertValue(uuid); + return; + } + if (!isStringOrFixedString(type)) throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Cannot insert MessagePack string into column with type {}.", type->getName()); @@ -218,6 +235,15 @@ static void insertNull(IColumn & column, DataTypePtr type) assert_cast(column).insertDefault(); } +static void insertUUID(IColumn & column, DataTypePtr /*type*/, const char * value, size_t size) +{ + ReadBufferFromMemory buf(value, size); + UUID uuid; + readBinaryBigEndian(uuid.toUnderType().items[0], buf); + readBinaryBigEndian(uuid.toUnderType().items[1], buf); + assert_cast(column).insertValue(uuid); +} + bool MsgPackVisitor::visit_positive_integer(UInt64 value) // NOLINT { insertInteger(info_stack.top().column, info_stack.top().type, value); @@ -232,13 +258,13 @@ bool MsgPackVisitor::visit_negative_integer(Int64 value) // NOLINT bool MsgPackVisitor::visit_str(const char * value, size_t size) // NOLINT { - insertString(info_stack.top().column, info_stack.top().type, value, size); + insertString(info_stack.top().column, info_stack.top().type, value, size, false); return true; } bool MsgPackVisitor::visit_bin(const char * value, size_t size) // NOLINT { - insertString(info_stack.top().column, info_stack.top().type, value, size); + insertString(info_stack.top().column, info_stack.top().type, value, size, true); return true; } @@ -324,6 +350,18 @@ bool MsgPackVisitor::visit_nil() return true; } +bool MsgPackVisitor::visit_ext(const char * value, uint32_t size) +{ + int8_t type = *value; + if (*value == int8_t(MsgPackExtensionTypes::UUID)) + { + insertUUID(info_stack.top().column, info_stack.top().type, value + 1, size - 1); + return true; + } + + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unsupported MsgPack extension type: {%x}", type); +} + void MsgPackVisitor::parse_error(size_t, size_t) // NOLINT { throw Exception("Error occurred while parsing msgpack data.", ErrorCodes::INCORRECT_DATA); @@ -455,8 +493,13 @@ DataTypePtr MsgPackSchemaReader::getDataType(const msgpack::object & object) } case msgpack::type::object_type::NIL: return nullptr; - default: - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Msgpack type is not supported"); + case msgpack::type::object_type::EXT: + { + msgpack::object_ext object_ext = object.via.ext; + if (object_ext.type() == int8_t(MsgPackExtensionTypes::UUID)) + return std::make_shared(); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Msgpack extension type {%x} is not supported", object_ext.type()); + } } } diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h index dd5655c80fc..2298e35fed5 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h @@ -42,6 +42,7 @@ public: bool end_map_key(); bool start_map_value(); bool end_map_value(); + bool visit_ext(const char * value, uint32_t size); /// This function will be called if error occurs in parsing [[noreturn]] void parse_error(size_t parsed_offset, size_t error_offset); diff --git a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp index 36a8a62b39e..edec9774b5f 100644 --- a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.cpp @@ -5,6 +5,9 @@ #include #include +#include +#include + #include #include #include @@ -19,6 +22,8 @@ #include #include +#include + namespace DB { @@ -27,8 +32,8 @@ namespace ErrorCodes extern const int ILLEGAL_COLUMN; } -MsgPackRowOutputFormat::MsgPackRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_) - : IRowOutputFormat(header_, out_, params_), packer(out_) {} +MsgPackRowOutputFormat::MsgPackRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_) + : IRowOutputFormat(header_, out_, params_), packer(out_), format_settings(format_settings_) {} void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr data_type, size_t row_num) { @@ -164,6 +169,42 @@ void MsgPackRowOutputFormat::serializeField(const IColumn & column, DataTypePtr serializeField(*dict_column, dict_type, index); return; } + case TypeIndex::UUID: + { + const auto & uuid_column = assert_cast(column); + switch (format_settings.msgpack.output_uuid_representation) + { + case FormatSettings::MsgPackUUIDRepresentation::BIN: + { + WriteBufferFromOwnString buf; + writeBinary(uuid_column.getElement(row_num), buf); + StringRef uuid_bin = buf.stringRef(); + packer.pack_bin(uuid_bin.size); + packer.pack_bin_body(uuid_bin.data, uuid_bin.size); + return; + } + case FormatSettings::MsgPackUUIDRepresentation::STR: + { + WriteBufferFromOwnString buf; + writeText(uuid_column.getElement(row_num), buf); + StringRef uuid_text = buf.stringRef(); + packer.pack_str(uuid_text.size); + packer.pack_bin_body(uuid_text.data, uuid_text.size); + return; + } + case FormatSettings::MsgPackUUIDRepresentation::EXT: + { + WriteBufferFromOwnString buf; + UUID value = uuid_column.getElement(row_num); + writeBinaryBigEndian(value.toUnderType().items[0], buf); + writeBinaryBigEndian(value.toUnderType().items[1], buf); + StringRef uuid_ext = buf.stringRef(); + packer.pack_ext(sizeof(UUID), int8_t(MsgPackExtensionTypes::UUID)); + packer.pack_ext_body(uuid_ext.data, uuid_ext.size); + return; + } + } + } default: break; } @@ -186,9 +227,9 @@ void registerOutputFormatMsgPack(FormatFactory & factory) WriteBuffer & buf, const Block & sample, const RowOutputFormatParams & params, - const FormatSettings &) + const FormatSettings & settings) { - return std::make_shared(buf, sample, params); + return std::make_shared(buf, sample, params, settings); }); factory.markOutputFormatSupportsParallelFormatting("MsgPack"); } diff --git a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h index 17d055818e9..e2abbd588c4 100644 --- a/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowOutputFormat.h @@ -15,10 +15,10 @@ namespace DB { -class MsgPackRowOutputFormat : public IRowOutputFormat +class MsgPackRowOutputFormat final : public IRowOutputFormat { public: - MsgPackRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_); + MsgPackRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_); String getName() const override { return "MsgPackRowOutputFormat"; } @@ -28,6 +28,7 @@ private: void serializeField(const IColumn & column, DataTypePtr data_type, size_t row_num); msgpack::packer packer; + const FormatSettings format_settings; }; } diff --git a/src/Processors/Formats/Impl/NativeFormat.cpp b/src/Processors/Formats/Impl/NativeFormat.cpp index 19e2ede6b65..bd95cfd6376 100644 --- a/src/Processors/Formats/Impl/NativeFormat.cpp +++ b/src/Processors/Formats/Impl/NativeFormat.cpp @@ -15,21 +15,22 @@ namespace DB class NativeInputFormat final : public IInputFormat { public: - NativeInputFormat(ReadBuffer & buf, const Block & header) - : IInputFormat(header, buf) - , reader(buf, header, 0) {} + NativeInputFormat(ReadBuffer & buf, const Block & header_) + : IInputFormat(header_, buf) + , reader(std::make_unique(buf, header_, 0)) + , header(header_) {} String getName() const override { return "Native"; } void resetParser() override { IInputFormat::resetParser(); - reader.resetParser(); + reader->resetParser(); } Chunk generate() override { - auto block = reader.read(); + auto block = reader->read(); if (!block) return {}; @@ -40,8 +41,15 @@ public: return Chunk(block.getColumns(), num_rows); } + void setReadBuffer(ReadBuffer & in_) override + { + reader = std::make_unique(in_, header, 0); + IInputFormat::setReadBuffer(in_); + } + private: - NativeReader reader; + std::unique_ptr reader; + Block header; }; class NativeOutputFormat final : public IOutputFormat diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp index 213226c9d68..bfdb9de7d26 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.cpp @@ -2,17 +2,12 @@ #include #include #include -#include namespace DB { void ParallelParsingInputFormat::segmentatorThreadFunction(ThreadGroupStatusPtr thread_group) { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); if (thread_group) CurrentThread::attachTo(thread_group); @@ -59,12 +54,8 @@ void ParallelParsingInputFormat::segmentatorThreadFunction(ThreadGroupStatusPtr void ParallelParsingInputFormat::parserThreadFunction(ThreadGroupStatusPtr thread_group, size_t current_ticket_number) { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); if (thread_group) - CurrentThread::attachTo(thread_group); + CurrentThread::attachToIfDetached(thread_group); const auto parser_unit_number = current_ticket_number % processing_units.size(); auto & unit = processing_units[parser_unit_number]; diff --git a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h index 5efdaf1b0b7..b2d395d94fe 100644 --- a/src/Processors/Formats/Impl/ParallelParsingInputFormat.h +++ b/src/Processors/Formats/Impl/ParallelParsingInputFormat.h @@ -196,7 +196,19 @@ private: size_t segmentator_ticket_number{0}; size_t reader_ticket_number{0}; + /// Mutex for internal synchronization between threads std::mutex mutex; + + /// finishAndWait can be called concurrently from + /// multiple threads. Atomic flag is not enough + /// because if finishAndWait called before destructor it can check the flag + /// and destroy object immediately. + std::mutex finish_and_wait_mutex; + /// We don't use parsing_finished flag because it can be setup from multiple + /// place in code. For example in case of bad data. It doesn't mean that we + /// don't need to finishAndWait our class. + bool finish_and_wait_called = false; + std::condition_variable reader_condvar; std::condition_variable segmentator_condvar; @@ -227,7 +239,7 @@ private: struct ProcessingUnit { - explicit ProcessingUnit() + ProcessingUnit() : status(ProcessingUnitStatus::READY_TO_INSERT) { } @@ -263,10 +275,21 @@ private: void finishAndWait() { + /// Defending concurrent segmentator thread join + std::lock_guard finish_and_wait_lock(finish_and_wait_mutex); + + /// We shouldn't execute this logic twice + if (finish_and_wait_called) + return; + + finish_and_wait_called = true; + + /// Signal background threads to finish parsing_finished = true; { - std::unique_lock lock(mutex); + /// Additionally notify condvars + std::lock_guard lock(mutex); segmentator_condvar.notify_all(); reader_condvar.notify_all(); } diff --git a/src/Processors/Formats/Impl/ProtobufRowInputFormat.h b/src/Processors/Formats/Impl/ProtobufRowInputFormat.h index d7d16d36ddf..9566cb45106 100644 --- a/src/Processors/Formats/Impl/ProtobufRowInputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowInputFormat.h @@ -26,7 +26,7 @@ class ProtobufSerializer; * INSERT INTO table FORMAT Protobuf SETTINGS format_schema = 'schema:Message' * where schema is the name of "schema.proto" file specifying protobuf schema. */ -class ProtobufRowInputFormat : public IRowInputFormat +class ProtobufRowInputFormat final : public IRowInputFormat { public: ProtobufRowInputFormat(ReadBuffer & in_, const Block & header_, const Params & params_, const FormatSchemaInfo & schema_info_, bool with_length_delimiter_); diff --git a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h index 97b727842a7..43d79b4d091 100644 --- a/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h +++ b/src/Processors/Formats/Impl/ProtobufRowOutputFormat.h @@ -26,7 +26,7 @@ struct FormatSettings; * SELECT * from table FORMAT Protobuf SETTINGS format_schema = 'schema:Message' * where schema is the name of "schema.proto" file specifying protobuf schema. */ -class ProtobufRowOutputFormat : public IRowOutputFormat +class ProtobufRowOutputFormat final : public IRowOutputFormat { public: ProtobufRowOutputFormat( diff --git a/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp b/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp index 91b1cc60fae..8bc9bb5e2a3 100644 --- a/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/RawBLOBRowInputFormat.cpp @@ -61,4 +61,3 @@ void registerRawBLOBSchemaReader(FormatFactory & factory) } } - diff --git a/src/Processors/Formats/Impl/RawBLOBRowInputFormat.h b/src/Processors/Formats/Impl/RawBLOBRowInputFormat.h index 367ca04f9d8..6fc1f277015 100644 --- a/src/Processors/Formats/Impl/RawBLOBRowInputFormat.h +++ b/src/Processors/Formats/Impl/RawBLOBRowInputFormat.h @@ -13,7 +13,7 @@ class ReadBuffer; /// This format slurps all input data into single value. /// This format can only parse a table with single field of type String or similar. -class RawBLOBRowInputFormat : public IRowInputFormat +class RawBLOBRowInputFormat final : public IRowInputFormat { public: RawBLOBRowInputFormat(const Block & header_, ReadBuffer & in_, Params params_); @@ -34,4 +34,3 @@ public: }; } - diff --git a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h index 2c34595c1a4..f6c4f0a58ca 100644 --- a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h +++ b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h @@ -24,7 +24,7 @@ class WriteBuffer; * * If you are output more than one value, the output format is ambiguous and you may not be able to read data back. */ -class RawBLOBRowOutputFormat : public IRowOutputFormat +class RawBLOBRowOutputFormat final : public IRowOutputFormat { public: RawBLOBRowOutputFormat( @@ -39,4 +39,3 @@ private: }; } - diff --git a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp index 90db6f6f0ec..4754b70d375 100644 --- a/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/RegexpRowInputFormat.cpp @@ -37,7 +37,7 @@ bool RegexpFieldExtractor::parseRow(PeekableReadBuffer & buf) do { - char * pos = find_first_symbols<'\n', '\r'>(buf.position(), buf.buffer().end()); + char * pos = find_first_symbols<'\n'>(buf.position(), buf.buffer().end()); line_size += pos - buf.position(); buf.position() = pos; } while (buf.position() == buf.buffer().end() && !buf.eof()); @@ -45,15 +45,19 @@ bool RegexpFieldExtractor::parseRow(PeekableReadBuffer & buf) buf.makeContinuousMemoryFromCheckpointToPos(); buf.rollbackToCheckpoint(); - bool match = RE2::FullMatchN(re2::StringPiece(buf.position(), line_size), regexp, re2_arguments_ptrs.data(), re2_arguments_ptrs.size()); + /// Allow DOS line endings. + size_t line_to_match = line_size; + if (line_size > 0 && buf.position()[line_size - 1] == '\r') + --line_to_match; + + bool match = re2_st::RE2::FullMatchN(re2_st::StringPiece(buf.position(), line_to_match), regexp, re2_arguments_ptrs.data(), re2_arguments_ptrs.size()); if (!match && !skip_unmatched) - throw Exception("Line \"" + std::string(buf.position(), line_size) + "\" doesn't match the regexp.", ErrorCodes::INCORRECT_DATA); + throw Exception("Line \"" + std::string(buf.position(), line_to_match) + "\" doesn't match the regexp.", ErrorCodes::INCORRECT_DATA); buf.position() += line_size; - checkChar('\r', buf); if (!buf.eof() && !checkChar('\n', buf)) - throw Exception("No \\n after \\r at the end of line.", ErrorCodes::INCORRECT_DATA); + throw Exception("No \\n at the end of line.", ErrorCodes::LOGICAL_ERROR); return match; } @@ -65,12 +69,12 @@ RegexpRowInputFormat::RegexpRowInputFormat( } RegexpRowInputFormat::RegexpRowInputFormat( - std::unique_ptr buf_, const Block & header_, Params params_, const FormatSettings & format_settings_) - : IRowInputFormat(header_, *buf_, std::move(params_)) - , buf(std::move(buf_)) - , format_settings(format_settings_) - , escaping_rule(format_settings_.regexp.escaping_rule) - , field_extractor(RegexpFieldExtractor(format_settings_)) + std::unique_ptr buf_, const Block & header_, Params params_, const FormatSettings & format_settings_) + : IRowInputFormat(header_, *buf_, std::move(params_)) + , buf(std::move(buf_)) + , format_settings(format_settings_) + , escaping_rule(format_settings_.regexp.escaping_rule) + , field_extractor(RegexpFieldExtractor(format_settings_)) { } @@ -174,20 +178,12 @@ static std::pair fileSegmentationEngineRegexpImpl(ReadBuffer & in, while (loadAtPosition(in, memory, pos) && need_more_data) { - pos = find_first_symbols<'\n', '\r'>(pos, in.buffer().end()); + pos = find_first_symbols<'\n'>(pos, in.buffer().end()); if (pos > in.buffer().end()) - throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR); else if (pos == in.buffer().end()) continue; - // Support DOS-style newline ("\r\n") - if (*pos == '\r') - { - ++pos; - if (pos == in.buffer().end()) - loadAtPosition(in, memory, pos); - } - if (memory.size() + static_cast(pos - in.position()) >= min_chunk_size) need_more_data = false; diff --git a/src/Processors/Formats/Impl/RegexpRowInputFormat.h b/src/Processors/Formats/Impl/RegexpRowInputFormat.h index dffd2f82e02..75c630d0607 100644 --- a/src/Processors/Formats/Impl/RegexpRowInputFormat.h +++ b/src/Processors/Formats/Impl/RegexpRowInputFormat.h @@ -1,7 +1,7 @@ #pragma once -#include -#include +#include +#include #include #include #include @@ -12,6 +12,7 @@ #include #include + namespace DB { @@ -26,17 +27,17 @@ public: /// Return true if row was successfully parsed and row fields were extracted. bool parseRow(PeekableReadBuffer & buf); - re2::StringPiece getField(size_t index) { return matched_fields[index]; } + re2_st::StringPiece getField(size_t index) { return matched_fields[index]; } size_t getMatchedFieldsSize() const { return matched_fields.size(); } size_t getNumberOfGroups() const { return regexp.NumberOfCapturingGroups(); } private: - const RE2 regexp; + const re2_st::RE2 regexp; // The vector of fields extracted from line using regexp. - std::vector matched_fields; + std::vector matched_fields; // These two vectors are needed to use RE2::FullMatchN (function for extracting fields). - std::vector re2_arguments; - std::vector re2_arguments_ptrs; + std::vector re2_arguments; + std::vector re2_arguments_ptrs; bool skip_unmatched; }; @@ -47,7 +48,7 @@ private: /// (according to format_regexp_escaping_rule setting). If the regexp did not match the line, /// if format_regexp_skip_unmatched is 1, the line is silently skipped, if the setting is 0, exception will be thrown. -class RegexpRowInputFormat : public IRowInputFormat +class RegexpRowInputFormat final : public IRowInputFormat { public: RegexpRowInputFormat(ReadBuffer & in_, const Block & header_, Params params_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp index 8a56c2ed5c7..f63d6fa9c46 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.cpp @@ -20,7 +20,7 @@ namespace ErrorCodes TSKVRowInputFormat::TSKVRowInputFormat(ReadBuffer & in_, Block header_, Params params_, const FormatSettings & format_settings_) - : IRowInputFormat(std::move(header_), in_, std::move(params_)), format_settings(format_settings_), name_map(header_.columns()) + : IRowInputFormat(std::move(header_), in_, std::move(params_)), format_settings(format_settings_), name_map(getPort().getHeader().columns()) { const auto & sample_block = getPort().getHeader(); size_t num_columns = sample_block.columns(); diff --git a/src/Processors/Formats/Impl/TSKVRowInputFormat.h b/src/Processors/Formats/Impl/TSKVRowInputFormat.h index 6aef50a0f84..3f708355b85 100644 --- a/src/Processors/Formats/Impl/TSKVRowInputFormat.h +++ b/src/Processors/Formats/Impl/TSKVRowInputFormat.h @@ -21,7 +21,7 @@ class ReadBuffer; * An equal sign can be escaped in the field name. * Also, as an additional element there may be a useless tskv fragment - it needs to be ignored. */ -class TSKVRowInputFormat : public IRowInputFormat +class TSKVRowInputFormat final : public IRowInputFormat { public: TSKVRowInputFormat(ReadBuffer & in_, Block header_, Params params_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/TSKVRowOutputFormat.h b/src/Processors/Formats/Impl/TSKVRowOutputFormat.h index 980e36c7e25..e9f9071f906 100644 --- a/src/Processors/Formats/Impl/TSKVRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TSKVRowOutputFormat.h @@ -11,7 +11,7 @@ namespace DB * TSKV is similar to TabSeparated, but before every value, its name and equal sign are specified: name=value. * This format is very inefficient. */ -class TSKVRowOutputFormat: public TabSeparatedRowOutputFormat +class TSKVRowOutputFormat final : public TabSeparatedRowOutputFormat { public: TSKVRowOutputFormat(WriteBuffer & out_, const Block & header, const RowOutputFormatParams & params_, const FormatSettings & format_settings); diff --git a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h index 1f2bfc255b8..ed67a8256bc 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowInputFormat.h @@ -11,7 +11,7 @@ namespace DB /** A stream to input data in tsv format. */ -class TabSeparatedRowInputFormat : public RowInputFormatWithNamesAndTypes +class TabSeparatedRowInputFormat final : public RowInputFormatWithNamesAndTypes { public: /** with_names - the first line is the header with the names of the columns @@ -28,7 +28,7 @@ private: bool isGarbageAfterField(size_t, ReadBuffer::Position pos) override { return *pos != '\n' && *pos != '\t'; } }; -class TabSeparatedFormatReader : public FormatWithNamesAndTypesReader +class TabSeparatedFormatReader final : public FormatWithNamesAndTypesReader { public: TabSeparatedFormatReader(ReadBuffer & in_, const FormatSettings & format_settings, bool is_raw_); diff --git a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h index eeada54d74e..8aac94812e2 100644 --- a/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h +++ b/src/Processors/Formats/Impl/TabSeparatedRowOutputFormat.h @@ -34,10 +34,10 @@ public: protected: void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; - void writeFieldDelimiter() override; + void writeFieldDelimiter() override final; void writeRowEndDelimiter() override; - void writeBeforeTotals() override; - void writeBeforeExtremes() override; + void writeBeforeTotals() override final; + void writeBeforeExtremes() override final; void writePrefix() override; void writeLine(const std::vector & values); diff --git a/src/Processors/Formats/Impl/TemplateRowInputFormat.h b/src/Processors/Formats/Impl/TemplateRowInputFormat.h index 755ad6cb39b..b5ced707ace 100644 --- a/src/Processors/Formats/Impl/TemplateRowInputFormat.h +++ b/src/Processors/Formats/Impl/TemplateRowInputFormat.h @@ -15,7 +15,7 @@ namespace DB class TemplateFormatReader; -class TemplateRowInputFormat : public RowInputFormatWithDiagnosticInfo +class TemplateRowInputFormat final : public RowInputFormatWithDiagnosticInfo { using EscapingRule = FormatSettings::EscapingRule; public: diff --git a/src/Processors/Formats/Impl/ValuesRowOutputFormat.h b/src/Processors/Formats/Impl/ValuesRowOutputFormat.h index 8d89854d43c..76c0a1e7873 100644 --- a/src/Processors/Formats/Impl/ValuesRowOutputFormat.h +++ b/src/Processors/Formats/Impl/ValuesRowOutputFormat.h @@ -12,7 +12,7 @@ class WriteBuffer; /** A stream for outputting data in the VALUES format (as in the INSERT request). */ -class ValuesRowOutputFormat : public IRowOutputFormat +class ValuesRowOutputFormat final : public IRowOutputFormat { public: ValuesRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h index 037aa183659..796c60d1cb1 100644 --- a/src/Processors/Formats/Impl/VerticalRowOutputFormat.h +++ b/src/Processors/Formats/Impl/VerticalRowOutputFormat.h @@ -15,7 +15,7 @@ class Context; /** Stream to output data in format "each value in separate row". * Usable to show few rows with many columns. */ -class VerticalRowOutputFormat : public IRowOutputFormat +class VerticalRowOutputFormat final : public IRowOutputFormat { public: VerticalRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_); diff --git a/src/Processors/Formats/Impl/XMLRowOutputFormat.h b/src/Processors/Formats/Impl/XMLRowOutputFormat.h index 04d81f0c2e1..abec25efbb9 100644 --- a/src/Processors/Formats/Impl/XMLRowOutputFormat.h +++ b/src/Processors/Formats/Impl/XMLRowOutputFormat.h @@ -13,7 +13,7 @@ namespace DB /** A stream for outputting data in XML format. */ -class XMLRowOutputFormat : public IRowOutputFormat +class XMLRowOutputFormat final : public IRowOutputFormat { public: XMLRowOutputFormat(WriteBuffer & out_, const Block & header_, const RowOutputFormatParams & params_, const FormatSettings & format_settings_); diff --git a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp index 6b2ee1c8039..af31ef01fcd 100644 --- a/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp +++ b/src/Processors/Merges/Algorithms/AggregatingSortedAlgorithm.cpp @@ -292,7 +292,7 @@ void AggregatingSortedAlgorithm::AggregatingMergedData::initAggregateDescription AggregatingSortedAlgorithm::AggregatingSortedAlgorithm( const Block & header, size_t num_inputs, SortDescription description_, size_t max_block_size) - : IMergingAlgorithmWithDelayedChunk(num_inputs, std::move(description_)) + : IMergingAlgorithmWithDelayedChunk(num_inputs, description_) , columns_definition(defineColumns(header, description_)) , merged_data(getMergedColumns(header, columns_definition), max_block_size, columns_definition) { diff --git a/src/Processors/Merges/Algorithms/Graphite.cpp b/src/Processors/Merges/Algorithms/Graphite.cpp index 38d3fa30b42..2c6d08ed287 100644 --- a/src/Processors/Merges/Algorithms/Graphite.cpp +++ b/src/Processors/Merges/Algorithms/Graphite.cpp @@ -10,9 +10,11 @@ #include #include +#include #include + using namespace std::literals; namespace DB::ErrorCodes @@ -286,7 +288,7 @@ std::string buildTaggedRegex(std::string regexp_str) else regexp_str = "[\\?&]"; - std::sort(std::begin(tags), std::end(tags)); /* sorted tag keys */ + ::sort(std::begin(tags), std::end(tags)); /* sorted tag keys */ regexp_str += fmt::format( "{}{}", fmt::join(tags, "&(.*&)?"), @@ -419,7 +421,7 @@ appendGraphitePattern( /// retention should be in descending order of age. if (pattern.type & pattern.TypeRetention) /// TypeRetention or TypeAll - std::sort(pattern.retentions.begin(), pattern.retentions.end(), compareRetentions); + ::sort(pattern.retentions.begin(), pattern.retentions.end(), compareRetentions); patterns.emplace_back(pattern); return patterns.back(); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index a859099de6c..9a9a71f9688 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include namespace ProfileEvents @@ -366,7 +367,6 @@ static ActionsDAGPtr createProjection(const Block & header) Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts_with_ranges, const Names & column_names, - const ActionsDAGPtr & sorting_key_prefix_expr, ActionsDAGPtr & out_projection, const InputOrderInfoPtr & input_order_info) { @@ -508,10 +508,19 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder( if (need_preliminary_merge) { + size_t fixed_prefix_size = input_order_info->order_key_fixed_prefix_descr.size(); + size_t prefix_size = fixed_prefix_size + input_order_info->order_key_prefix_descr.size(); + + auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone(); + order_key_prefix_ast->children.resize(prefix_size); + + auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); + auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false); + const auto & sorting_columns = metadata_snapshot->getSortingKey().column_names; + SortDescription sort_description; - for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j) - sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j], - input_order_info->direction, 1); + for (size_t j = 0; j < prefix_size; ++j) + sort_description.emplace_back(sorting_columns[j], input_order_info->direction); auto sorting_key_expr = std::make_shared(sorting_key_prefix_expr); @@ -911,6 +920,11 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( parts_before_pk = parts.size(); auto reader_settings = getMergeTreeReaderSettings(context); + + bool use_skip_indexes = settings.use_skip_indexes; + if (select.final() && !settings.use_skip_indexes_if_final) + use_skip_indexes = false; + result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes( std::move(parts), metadata_snapshot, @@ -921,7 +935,7 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead( log, num_streams, result.index_stats, - context->getSettings().use_skip_indexes); + use_skip_indexes); } catch (...) { @@ -1015,7 +1029,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons /// Skip this if final was used, because such columns were already added from PK. std::vector add_columns = result.sampling.filter_expression->getRequiredColumns().getNames(); column_names_to_read.insert(column_names_to_read.end(), add_columns.begin(), add_columns.end()); - std::sort(column_names_to_read.begin(), column_names_to_read.end()); + ::sort(column_names_to_read.begin(), column_names_to_read.end()); column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); } @@ -1039,7 +1053,7 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons if (!data.merging_params.version_column.empty()) column_names_to_read.push_back(data.merging_params.version_column); - std::sort(column_names_to_read.begin(), column_names_to_read.end()); + ::sort(column_names_to_read.begin(), column_names_to_read.end()); column_names_to_read.erase(std::unique(column_names_to_read.begin(), column_names_to_read.end()), column_names_to_read.end()); pipe = spreadMarkRangesAmongStreamsFinal( @@ -1049,17 +1063,9 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons } else if ((settings.optimize_read_in_order || settings.optimize_aggregation_in_order) && input_order_info) { - size_t prefix_size = input_order_info->order_key_prefix_descr.size(); - auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone(); - order_key_prefix_ast->children.resize(prefix_size); - - auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical()); - auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false); - pipe = spreadMarkRangesAmongStreamsWithOrder( std::move(result.parts_with_ranges), column_names_to_read, - sorting_key_prefix_expr, result_projection, input_order_info); } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 0bdfa66bcc7..0d07a3e2ea2 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -173,7 +173,6 @@ private: Pipe spreadMarkRangesAmongStreamsWithOrder( RangesInDataParts && parts_with_ranges, const Names & column_names, - const ActionsDAGPtr & sorting_key_prefix_expr, ActionsDAGPtr & out_projection, const InputOrderInfoPtr & input_order_info); diff --git a/src/Processors/Transforms/AggregatingInOrderTransform.cpp b/src/Processors/Transforms/AggregatingInOrderTransform.cpp index 857f362c4be..63497ea1af4 100644 --- a/src/Processors/Transforms/AggregatingInOrderTransform.cpp +++ b/src/Processors/Transforms/AggregatingInOrderTransform.cpp @@ -121,7 +121,7 @@ void AggregatingInOrderTransform::consume(Chunk chunk) /// Add data to aggr. state if interval is not empty. Empty when haven't found current key in new block. if (key_begin != key_end) - params->aggregator.executeOnIntervalWithoutKeyImpl(variants.without_key, key_begin, key_end, aggregate_function_instructions.data(), variants.aggregates_pool); + params->aggregator.executeOnIntervalWithoutKeyImpl(variants, key_begin, key_end, aggregate_function_instructions.data(), variants.aggregates_pool); current_memory_usage = getCurrentMemoryUsage() - initial_memory_usage; diff --git a/src/Processors/Transforms/CreatingSetsTransform.h b/src/Processors/Transforms/CreatingSetsTransform.h index 839ab0cac88..48a32ea8663 100644 --- a/src/Processors/Transforms/CreatingSetsTransform.h +++ b/src/Processors/Transforms/CreatingSetsTransform.h @@ -44,8 +44,8 @@ public: private: SubqueryForSet subquery; - std::unique_ptr executor; QueryPipeline table_out; + std::unique_ptr executor; UInt64 read_rows = 0; Stopwatch watch; diff --git a/src/Processors/Transforms/FilterTransform.cpp b/src/Processors/Transforms/FilterTransform.cpp index 364fb8e1958..9164599f3b1 100644 --- a/src/Processors/Transforms/FilterTransform.cpp +++ b/src/Processors/Transforms/FilterTransform.cpp @@ -138,8 +138,6 @@ void FilterTransform::transform(Chunk & chunk) return; } - FilterDescription filter_and_holder(*filter_column); - /** Let's find out how many rows will be in result. * To do this, we filter out the first non-constant column * or calculate number of set bytes in the filter. @@ -154,14 +152,20 @@ void FilterTransform::transform(Chunk & chunk) } } + std::unique_ptr filter_description; + if (filter_column->isSparse()) + filter_description = std::make_unique(*filter_column); + else + filter_description = std::make_unique(*filter_column); + size_t num_filtered_rows = 0; if (first_non_constant_column != num_columns) { - columns[first_non_constant_column] = columns[first_non_constant_column]->filter(*filter_and_holder.data, -1); + columns[first_non_constant_column] = filter_description->filter(*columns[first_non_constant_column], -1); num_filtered_rows = columns[first_non_constant_column]->size(); } else - num_filtered_rows = countBytesInFilter(*filter_and_holder.data); + num_filtered_rows = filter_description->countBytesInFilter(); /// If the current block is completely filtered out, let's move on to the next one. if (num_filtered_rows == 0) @@ -207,7 +211,7 @@ void FilterTransform::transform(Chunk & chunk) if (isColumnConst(*current_column)) current_column = current_column->cut(0, num_filtered_rows); else - current_column = current_column->filter(*filter_and_holder.data, num_filtered_rows); + current_column = filter_description->filter(*current_column, num_filtered_rows); } chunk.setColumns(std::move(columns), num_filtered_rows); diff --git a/src/Processors/Transforms/MergingAggregatedTransform.cpp b/src/Processors/Transforms/MergingAggregatedTransform.cpp index 37419f55aae..dd2b315d53c 100644 --- a/src/Processors/Transforms/MergingAggregatedTransform.cpp +++ b/src/Processors/Transforms/MergingAggregatedTransform.cpp @@ -1,5 +1,6 @@ #include #include +#include namespace DB { @@ -34,21 +35,30 @@ void MergingAggregatedTransform::consume(Chunk chunk) if (!info) throw Exception("Chunk info was not set for chunk in MergingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); - const auto * agg_info = typeid_cast(info.get()); - if (!agg_info) + if (const auto * agg_info = typeid_cast(info.get())) + { + /** If the remote servers used a two-level aggregation method, + * then blocks will contain information about the number of the bucket. + * Then the calculations can be parallelized by buckets. + * We decompose the blocks to the bucket numbers indicated in them. + */ + + auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); + block.info.is_overflows = agg_info->is_overflows; + block.info.bucket_num = agg_info->bucket_num; + + bucket_to_blocks[agg_info->bucket_num].emplace_back(std::move(block)); + } + else if (const auto * in_order_info = typeid_cast(info.get())) + { + auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); + block.info.is_overflows = false; + block.info.bucket_num = -1; + + bucket_to_blocks[block.info.bucket_num].emplace_back(std::move(block)); + } + else throw Exception("Chunk should have AggregatedChunkInfo in MergingAggregatedTransform.", ErrorCodes::LOGICAL_ERROR); - - /** If the remote servers used a two-level aggregation method, - * then blocks will contain information about the number of the bucket. - * Then the calculations can be parallelized by buckets. - * We decompose the blocks to the bucket numbers indicated in them. - */ - - auto block = getInputPort().getHeader().cloneWithColumns(chunk.getColumns()); - block.info.is_overflows = agg_info->is_overflows; - block.info.bucket_num = agg_info->bucket_num; - - bucket_to_blocks[agg_info->bucket_num].emplace_back(std::move(block)); } Chunk MergingAggregatedTransform::generate() diff --git a/src/Processors/Transforms/buildPushingToViewsChain.cpp b/src/Processors/Transforms/buildPushingToViewsChain.cpp index 17075e2b318..19302afb5c9 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.cpp +++ b/src/Processors/Transforms/buildPushingToViewsChain.cpp @@ -331,7 +331,7 @@ Chain buildPushingToViewsChain( { auto executing_inner_query = std::make_shared( storage_header, views_data->views.back(), views_data); - executing_inner_query->setRuntimeData(view_thread_status, elapsed_counter_ms); + executing_inner_query->setRuntimeData(view_thread_status, view_counter_ms); out.addSource(std::move(executing_inner_query)); } @@ -381,7 +381,7 @@ Chain buildPushingToViewsChain( processors.emplace_front(std::move(copying_data)); processors.emplace_back(std::move(finalizing_views)); result_chain = Chain(std::move(processors)); - result_chain.setNumThreads(max_parallel_streams); + result_chain.setNumThreads(std::min(views_data->max_threads, max_parallel_streams)); } if (auto * live_view = dynamic_cast(storage.get())) diff --git a/src/Processors/Transforms/buildPushingToViewsChain.h b/src/Processors/Transforms/buildPushingToViewsChain.h index 260fdfb3a19..98e7f19a37a 100644 --- a/src/Processors/Transforms/buildPushingToViewsChain.h +++ b/src/Processors/Transforms/buildPushingToViewsChain.h @@ -18,7 +18,7 @@ namespace DB struct ViewRuntimeData { - /// A query we should run over inserted block befire pushing into inner storage. + /// A query we should run over inserted block before pushing into inner storage. const ASTPtr query; /// This structure is expected by inner storage. Will convert query result to it. Block sample_block; diff --git a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp index 64c8a01bb9c..4ee3f2d4b82 100644 --- a/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp +++ b/src/Processors/Transforms/getSourceFromASTInsertQuery.cpp @@ -59,7 +59,7 @@ InputFormatPtr getInputFormatFromASTInsertQuery( : std::make_unique(); /// Create a source from input buffer using format from query - auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettings().max_insert_block_size); + auto source = context->getInputFormat(ast_insert_query->format, *input_buffer, header, context->getSettingsRef().max_insert_block_size); source->addBuffer(std::move(input_buffer)); return source; } diff --git a/src/QueryPipeline/BlockIO.h b/src/QueryPipeline/BlockIO.h index 5918b4b27fc..748e46c3a1e 100644 --- a/src/QueryPipeline/BlockIO.h +++ b/src/QueryPipeline/BlockIO.h @@ -40,10 +40,12 @@ struct BlockIO pipeline.reset(); } - void onException() const + void onException() { if (exception_callback) exception_callback(); + + pipeline.reset(); } private: diff --git a/src/QueryPipeline/RemoteInserter.cpp b/src/QueryPipeline/RemoteInserter.cpp index c34c625dc6d..13d087f0db9 100644 --- a/src/QueryPipeline/RemoteInserter.cpp +++ b/src/QueryPipeline/RemoteInserter.cpp @@ -24,7 +24,9 @@ RemoteInserter::RemoteInserter( const String & query_, const Settings & settings_, const ClientInfo & client_info_) - : connection(connection_), query(query_) + : connection(connection_) + , query(query_) + , server_revision(connection.getServerRevision(timeouts)) { ClientInfo modified_client_info = client_info_; modified_client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; diff --git a/src/QueryPipeline/RemoteInserter.h b/src/QueryPipeline/RemoteInserter.h index 0688b555825..5b5de962cc6 100644 --- a/src/QueryPipeline/RemoteInserter.h +++ b/src/QueryPipeline/RemoteInserter.h @@ -35,12 +35,14 @@ public: ~RemoteInserter(); const Block & getHeader() const { return header; } + UInt64 getServerRevision() const { return server_revision; } private: Connection & connection; String query; Block header; bool finished = false; + UInt64 server_revision; }; } diff --git a/src/Server/CertificateReloader.cpp b/src/Server/CertificateReloader.cpp new file mode 100644 index 00000000000..f3f366876da --- /dev/null +++ b/src/Server/CertificateReloader.cpp @@ -0,0 +1,136 @@ +#include "CertificateReloader.h" + +#if USE_SSL + +#include +#include +#include +#include +#include + + +namespace DB +{ + +namespace +{ +/// Call set process for certificate. +int callSetCertificate(SSL * ssl, [[maybe_unused]] void * arg) +{ + return CertificateReloader::instance().setCertificate(ssl); +} + +} + + +namespace ErrorCodes +{ + extern const int CANNOT_STAT; +} + + +/// This is callback for OpenSSL. It will be called on every connection to obtain a certificate and private key. +int CertificateReloader::setCertificate(SSL * ssl) +{ + auto current = data.get(); + if (!current) + return -1; + + SSL_use_certificate(ssl, const_cast(current->cert.certificate())); + SSL_use_RSAPrivateKey(ssl, current->key.impl()->getRSA()); + + int err = SSL_check_private_key(ssl); + if (err != 1) + { + std::string msg = Poco::Net::Utility::getLastError(); + LOG_ERROR(log, "Unusable key-pair {}", msg); + return -1; + } + + return 1; +} + + +void CertificateReloader::init() +{ + LOG_DEBUG(log, "Initializing certificate reloader."); + + /// Set a callback for OpenSSL to allow get the updated cert and key. + + auto* ctx = Poco::Net::SSLManager::instance().defaultServerContext()->sslContext(); + SSL_CTX_set_cert_cb(ctx, callSetCertificate, nullptr); + init_was_not_made = false; +} + + +void CertificateReloader::tryLoad(const Poco::Util::AbstractConfiguration & config) +{ + /// If at least one of the files is modified - recreate + + std::string new_cert_path = config.getString("openSSL.server.certificateFile", ""); + std::string new_key_path = config.getString("openSSL.server.privateKeyFile", ""); + + /// For empty paths (that means, that user doesn't want to use certificates) + /// no processing required + + if (new_cert_path.empty() || new_key_path.empty()) + { + LOG_INFO(log, "One of paths is empty. Cannot apply new configuration for certificates. Fill all paths and try again."); + } + else + { + bool cert_file_changed = cert_file.changeIfModified(std::move(new_cert_path), log); + bool key_file_changed = key_file.changeIfModified(std::move(new_key_path), log); + + if (cert_file_changed || key_file_changed) + { + LOG_DEBUG(log, "Reloading certificate ({}) and key ({}).", cert_file.path, key_file.path); + data.set(std::make_unique(cert_file.path, key_file.path)); + LOG_INFO(log, "Reloaded certificate ({}) and key ({}).", cert_file.path, key_file.path); + } + + /// If callback is not set yet + try + { + if (init_was_not_made) + init(); + } + catch (...) + { + init_was_not_made = true; + LOG_ERROR(log, fmt::runtime(getCurrentExceptionMessage(false))); + } + } +} + + +CertificateReloader::Data::Data(std::string cert_path, std::string key_path) + : cert(cert_path), key(/* public key */ "", /* private key */ key_path) +{ +} + + +bool CertificateReloader::File::changeIfModified(std::string new_path, Poco::Logger * logger) +{ + std::error_code ec; + std::filesystem::file_time_type new_modification_time = std::filesystem::last_write_time(new_path, ec); + if (ec) + { + LOG_ERROR(logger, "Cannot obtain modification time for {} file {}, skipping update. {}", + description, new_path, errnoToString(ErrorCodes::CANNOT_STAT, ec.value())); + return false; + } + + if (new_path != path || new_modification_time != modification_time) + { + path = new_path; + modification_time = new_modification_time; + return true; + } + + return false; +} + +} + +#endif diff --git a/src/Server/CertificateReloader.h b/src/Server/CertificateReloader.h new file mode 100644 index 00000000000..7f93b006875 --- /dev/null +++ b/src/Server/CertificateReloader.h @@ -0,0 +1,88 @@ +#pragma once + +#if !defined(ARCADIA_BUILD) +# include +#endif + +#if USE_SSL + +#include +#include + +#include +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +/// The CertificateReloader singleton performs 2 functions: +/// 1. Dynamic reloading of TLS key-pair when requested by server: +/// Server config reloader notifies CertificateReloader when the config changes. +/// On changed config, CertificateReloader reloads certs from disk. +/// 2. Implement `SSL_CTX_set_cert_cb` to set certificate for a new connection: +/// OpenSSL invokes a callback to setup a connection. +class CertificateReloader +{ +public: + using stat_t = struct stat; + + /// Singleton + CertificateReloader(CertificateReloader const &) = delete; + void operator=(CertificateReloader const &) = delete; + static CertificateReloader & instance() + { + static CertificateReloader instance; + return instance; + } + + /// Initialize the callback and perform the initial cert loading + void init(); + + /// Handle configuration reload + void tryLoad(const Poco::Util::AbstractConfiguration & config); + + /// A callback for OpenSSL + int setCertificate(SSL * ssl); + +private: + CertificateReloader() + { + } + + Poco::Logger * log = &Poco::Logger::get("CertificateReloader"); + + struct File + { + const char * description; + File(const char * description_) : description(description_) {} + + std::string path; + std::filesystem::file_time_type modification_time; + + bool changeIfModified(std::string new_path, Poco::Logger * logger); + }; + + File cert_file{"certificate"}; + File key_file{"key"}; + + struct Data + { + Poco::Crypto::X509Certificate cert; + Poco::Crypto::RSAKey key; + + Data(std::string cert_path, std::string key_path); + }; + + MultiVersion data; + bool init_was_not_made = true; +}; + +} + +#endif diff --git a/src/Server/GRPCServer.cpp b/src/Server/GRPCServer.cpp index 589bdd63f41..eeaf5b32a92 100644 --- a/src/Server/GRPCServer.cpp +++ b/src/Server/GRPCServer.cpp @@ -9,7 +9,6 @@ #include #include #include -#include #include #include #include @@ -52,6 +51,7 @@ using GRPCQueryInfo = clickhouse::grpc::QueryInfo; using GRPCResult = clickhouse::grpc::Result; using GRPCException = clickhouse::grpc::Exception; using GRPCProgress = clickhouse::grpc::Progress; +using GRPCObsoleteTransportCompression = clickhouse::grpc::ObsoleteTransportCompression; namespace DB { @@ -64,6 +64,7 @@ namespace ErrorCodes extern const int NETWORK_ERROR; extern const int NO_DATA_TO_INSERT; extern const int SUPPORT_IS_DISABLED; + extern const int BAD_REQUEST_PARAMETER; } namespace @@ -101,62 +102,6 @@ namespace }); } - grpc_compression_algorithm parseCompressionAlgorithm(const String & str) - { - if (str == "none") - return GRPC_COMPRESS_NONE; - else if (str == "deflate") - return GRPC_COMPRESS_DEFLATE; - else if (str == "gzip") - return GRPC_COMPRESS_GZIP; - else if (str == "stream_gzip") - return GRPC_COMPRESS_STREAM_GZIP; - else - throw Exception("Unknown compression algorithm: '" + str + "'", ErrorCodes::INVALID_CONFIG_PARAMETER); - } - - grpc_compression_level parseCompressionLevel(const String & str) - { - if (str == "none") - return GRPC_COMPRESS_LEVEL_NONE; - else if (str == "low") - return GRPC_COMPRESS_LEVEL_LOW; - else if (str == "medium") - return GRPC_COMPRESS_LEVEL_MED; - else if (str == "high") - return GRPC_COMPRESS_LEVEL_HIGH; - else - throw Exception("Unknown compression level: '" + str + "'", ErrorCodes::INVALID_CONFIG_PARAMETER); - } - - grpc_compression_algorithm convertCompressionAlgorithm(const ::clickhouse::grpc::CompressionAlgorithm & algorithm) - { - if (algorithm == ::clickhouse::grpc::NO_COMPRESSION) - return GRPC_COMPRESS_NONE; - else if (algorithm == ::clickhouse::grpc::DEFLATE) - return GRPC_COMPRESS_DEFLATE; - else if (algorithm == ::clickhouse::grpc::GZIP) - return GRPC_COMPRESS_GZIP; - else if (algorithm == ::clickhouse::grpc::STREAM_GZIP) - return GRPC_COMPRESS_STREAM_GZIP; - else - throw Exception("Unknown compression algorithm: '" + ::clickhouse::grpc::CompressionAlgorithm_Name(algorithm) + "'", ErrorCodes::INVALID_GRPC_QUERY_INFO); - } - - grpc_compression_level convertCompressionLevel(const ::clickhouse::grpc::CompressionLevel & level) - { - if (level == ::clickhouse::grpc::COMPRESSION_NONE) - return GRPC_COMPRESS_LEVEL_NONE; - else if (level == ::clickhouse::grpc::COMPRESSION_LOW) - return GRPC_COMPRESS_LEVEL_LOW; - else if (level == ::clickhouse::grpc::COMPRESSION_MEDIUM) - return GRPC_COMPRESS_LEVEL_MED; - else if (level == ::clickhouse::grpc::COMPRESSION_HIGH) - return GRPC_COMPRESS_LEVEL_HIGH; - else - throw Exception("Unknown compression level: '" + ::clickhouse::grpc::CompressionLevel_Name(level) + "'", ErrorCodes::INVALID_GRPC_QUERY_INFO); - } - /// Gets file's contents as a string, throws an exception if failed. String readFile(const String & filepath) { @@ -193,6 +138,102 @@ namespace return grpc::InsecureServerCredentials(); } + /// Transport compression makes gRPC library to compress packed Result messages before sending them through network. + struct TransportCompression + { + grpc_compression_algorithm algorithm; + grpc_compression_level level; + + /// Extracts the settings of transport compression from a query info if possible. + static std::optional fromQueryInfo(const GRPCQueryInfo & query_info) + { + TransportCompression res; + if (!query_info.transport_compression_type().empty()) + { + res.setAlgorithm(query_info.transport_compression_type(), ErrorCodes::INVALID_GRPC_QUERY_INFO); + res.setLevel(query_info.transport_compression_level(), ErrorCodes::INVALID_GRPC_QUERY_INFO); + return res; + } + + if (query_info.has_obsolete_result_compression()) + { + switch (query_info.obsolete_result_compression().algorithm()) + { + case GRPCObsoleteTransportCompression::NO_COMPRESSION: res.algorithm = GRPC_COMPRESS_NONE; break; + case GRPCObsoleteTransportCompression::DEFLATE: res.algorithm = GRPC_COMPRESS_DEFLATE; break; + case GRPCObsoleteTransportCompression::GZIP: res.algorithm = GRPC_COMPRESS_GZIP; break; + case GRPCObsoleteTransportCompression::STREAM_GZIP: res.algorithm = GRPC_COMPRESS_STREAM_GZIP; break; + default: throw Exception(ErrorCodes::INVALID_GRPC_QUERY_INFO, "Unknown compression algorithm: {}", GRPCObsoleteTransportCompression::CompressionAlgorithm_Name(query_info.obsolete_result_compression().algorithm())); + } + + switch (query_info.obsolete_result_compression().level()) + { + case GRPCObsoleteTransportCompression::COMPRESSION_NONE: res.level = GRPC_COMPRESS_LEVEL_NONE; break; + case GRPCObsoleteTransportCompression::COMPRESSION_LOW: res.level = GRPC_COMPRESS_LEVEL_LOW; break; + case GRPCObsoleteTransportCompression::COMPRESSION_MEDIUM: res.level = GRPC_COMPRESS_LEVEL_MED; break; + case GRPCObsoleteTransportCompression::COMPRESSION_HIGH: res.level = GRPC_COMPRESS_LEVEL_HIGH; break; + default: throw Exception(ErrorCodes::INVALID_GRPC_QUERY_INFO, "Unknown compression level: {}", GRPCObsoleteTransportCompression::CompressionLevel_Name(query_info.obsolete_result_compression().level())); + } + return res; + } + + return std::nullopt; + } + + /// Extracts the settings of transport compression from the server configuration. + static TransportCompression fromConfiguration(const Poco::Util::AbstractConfiguration & config) + { + TransportCompression res; + if (config.has("grpc.transport_compression_type")) + { + res.setAlgorithm(config.getString("grpc.transport_compression_type"), ErrorCodes::INVALID_CONFIG_PARAMETER); + res.setLevel(config.getInt("grpc.transport_compression_level", 0), ErrorCodes::INVALID_CONFIG_PARAMETER); + } + else + { + res.setAlgorithm(config.getString("grpc.compression", "none"), ErrorCodes::INVALID_CONFIG_PARAMETER); + res.setLevel(config.getString("grpc.compression_level", "none"), ErrorCodes::INVALID_CONFIG_PARAMETER); + } + return res; + } + + private: + void setAlgorithm(const String & str, int error_code) + { + if (str == "none") + algorithm = GRPC_COMPRESS_NONE; + else if (str == "deflate") + algorithm = GRPC_COMPRESS_DEFLATE; + else if (str == "gzip") + algorithm = GRPC_COMPRESS_GZIP; + else if (str == "stream_gzip") + algorithm = GRPC_COMPRESS_STREAM_GZIP; + else + throw Exception(error_code, "Unknown compression algorithm: '{}'", str); + } + + void setLevel(const String & str, int error_code) + { + if (str == "none") + level = GRPC_COMPRESS_LEVEL_NONE; + else if (str == "low") + level = GRPC_COMPRESS_LEVEL_LOW; + else if (str == "medium") + level = GRPC_COMPRESS_LEVEL_MED; + else if (str == "high") + level = GRPC_COMPRESS_LEVEL_HIGH; + else + throw Exception(error_code, "Unknown compression level: '{}'", str); + } + + void setLevel(int level_, int error_code) + { + if (0 <= level_ && level_ < GRPC_COMPRESS_LEVEL_COUNT) + level = static_cast(level_); + else + throw Exception(error_code, "Compression level {} is out of range 0..{}", level_, GRPC_COMPRESS_LEVEL_COUNT - 1); + } + }; /// Gets session's timeout from query info or from the server config. std::chrono::steady_clock::duration getSessionTimeout(const GRPCQueryInfo & query_info, const Poco::Util::AbstractConfiguration & config) @@ -284,15 +325,19 @@ namespace return Poco::Net::SocketAddress{peer.substr(peer.find(':') + 1)}; } - void setResultCompression(grpc_compression_algorithm algorithm, grpc_compression_level level) + std::optional getClientHeader(const String & key) const { - grpc_context.set_compression_algorithm(algorithm); - grpc_context.set_compression_level(level); + const auto & client_metadata = grpc_context.client_metadata(); + auto it = client_metadata.find(key); + if (it != client_metadata.end()) + return String{it->second.data(), it->second.size()}; + return std::nullopt; } - void setResultCompression(const ::clickhouse::grpc::Compression & compression) + void setTransportCompression(const TransportCompression & transport_compression) { - setResultCompression(convertCompressionAlgorithm(compression.algorithm()), convertCompressionLevel(compression.level())); + grpc_context.set_compression_algorithm(transport_compression.algorithm); + grpc_context.set_compression_level(transport_compression.level); } protected: @@ -597,6 +642,9 @@ namespace void throwIfFailedToReadQueryInfo(); bool isQueryCancelled(); + void addQueryDetailsToResult(); + void addOutputFormatToResult(); + void addOutputColumnsNamesAndTypesToResult(const Block & headers); void addProgressToResult(); void addTotalsToResult(const Block & totals); void addExtremesToResult(const Block & extremes); @@ -619,10 +667,12 @@ namespace ASTInsertQuery * insert_query = nullptr; String input_format; String input_data_delimiter; + CompressionMethod input_compression_method = CompressionMethod::None; PODArray output; String output_format; - CompressionMethod compression_method = CompressionMethod::None; - int compression_level = 0; + bool send_output_columns_names_and_types = false; + CompressionMethod output_compression_method = CompressionMethod::None; + int output_compression_level = 0; uint64_t interactive_delay = 100000; bool send_exception_with_stacktrace = true; @@ -751,6 +801,23 @@ namespace session->authenticate(user, password, user_address); session->getClientInfo().quota_key = quota_key; + ClientInfo client_info = session->getClientInfo(); + + /// Parse the OpenTelemetry traceparent header. + auto traceparent = responder->getClientHeader("traceparent"); + if (traceparent) + { + String error; + if (!client_info.client_trace_context.parseTraceparentHeader(traceparent.value(), error)) + { + throw Exception(ErrorCodes::BAD_REQUEST_PARAMETER, + "Failed to parse OpenTelemetry traceparent header '{}': {}", + traceparent.value(), error); + } + auto tracestate = responder->getClientHeader("tracestate"); + client_info.client_trace_context.tracestate = tracestate.value_or(""); + } + /// The user could specify session identifier and session timeout. /// It allows to modify settings, create temporary tables and reuse them in subsequent requests. if (!query_info.session_id().empty()) @@ -759,7 +826,7 @@ namespace query_info.session_id(), getSessionTimeout(query_info, iserver.config()), query_info.session_check()); } - query_context = session->makeQueryContext(); + query_context = session->makeQueryContext(std::move(client_info)); /// Prepare settings. SettingsChanges settings_changes; @@ -789,9 +856,9 @@ namespace if (!query_info.database().empty()) query_context->setCurrentDatabase(query_info.database()); - /// Apply compression settings for this call. - if (query_info.has_result_compression()) - responder->setResultCompression(query_info.result_compression()); + /// Apply transport compression for this call. + if (auto transport_compression = TransportCompression::fromQueryInfo(query_info)) + responder->setTransportCompression(*transport_compression); /// The interactive delay will be used to show progress. interactive_delay = settings.interactive_delay; @@ -825,9 +892,19 @@ namespace if (output_format.empty()) output_format = query_context->getDefaultFormat(); + send_output_columns_names_and_types = query_info.send_output_columns(); + /// Choose compression. - compression_method = chooseCompressionMethod("", query_info.compression_type()); - compression_level = query_info.compression_level(); + String input_compression_method_str = query_info.input_compression_type(); + if (input_compression_method_str.empty()) + input_compression_method_str = query_info.obsolete_compression_type(); + input_compression_method = chooseCompressionMethod("", input_compression_method_str); + + String output_compression_method_str = query_info.output_compression_type(); + if (output_compression_method_str.empty()) + output_compression_method_str = query_info.obsolete_compression_type(); + output_compression_method = chooseCompressionMethod("", output_compression_method_str); + output_compression_level = query_info.output_compression_level(); /// Set callback to create and fill external tables query_context->setExternalTablesInitializer([this] (ContextPtr context) @@ -958,46 +1035,15 @@ namespace return {nullptr, 0}; /// no more input data }); - read_buffer = wrapReadBufferWithCompressionMethod(std::move(read_buffer), compression_method); + read_buffer = wrapReadBufferWithCompressionMethod(std::move(read_buffer), input_compression_method); assert(!pipeline); auto source = query_context->getInputFormat( input_format, *read_buffer, header, query_context->getSettings().max_insert_block_size); + QueryPipelineBuilder builder; builder.init(Pipe(source)); - /// Add default values if necessary. - if (ast) - { - if (insert_query) - { - auto table_id = StorageID::createEmpty(); - - if (insert_query->table_id) - { - table_id = query_context->resolveStorageID(insert_query->table_id, Context::ResolveOrdinary); - } - else - { - StorageID local_table_id(insert_query->getDatabase(), insert_query->getTable()); - table_id = query_context->resolveStorageID(local_table_id, Context::ResolveOrdinary); - } - - if (query_context->getSettingsRef().input_format_defaults_for_omitted_fields && table_id) - { - StoragePtr storage = DatabaseCatalog::instance().getTable(table_id, query_context); - const auto & columns = storage->getInMemoryMetadataPtr()->getColumns(); - if (!columns.empty()) - { - builder.addSimpleTransform([&](const Block & cur_header) - { - return std::make_shared(cur_header, columns, *source, query_context); - }); - } - } - } - } - pipeline = std::make_unique(QueryPipelineBuilder::getPipeline(std::move(builder))); pipeline_executor = std::make_unique(*pipeline); } @@ -1110,6 +1156,9 @@ namespace void Call::generateOutput() { + /// We add query_id and time_zone to the first result anyway. + addQueryDetailsToResult(); + if (!io.pipeline.initialized() || io.pipeline.pushing()) return; @@ -1117,13 +1166,13 @@ namespace if (io.pipeline.pulling()) header = io.pipeline.getHeader(); - if (compression_method != CompressionMethod::None) + if (output_compression_method != CompressionMethod::None) output.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data. write_buffer = std::make_unique>>(output); nested_write_buffer = static_cast> *>(write_buffer.get()); - if (compression_method != CompressionMethod::None) + if (output_compression_method != CompressionMethod::None) { - write_buffer = wrapWriteBufferWithCompressionMethod(std::move(write_buffer), compression_method, compression_level); + write_buffer = wrapWriteBufferWithCompressionMethod(std::move(write_buffer), output_compression_method, output_compression_level); compressing_write_buffer = write_buffer.get(); } @@ -1149,6 +1198,9 @@ namespace return true; }; + addOutputFormatToResult(); + addOutputColumnsNamesAndTypesToResult(header); + Block block; while (check_for_cancel()) { @@ -1232,7 +1284,7 @@ namespace { io.onException(); - LOG_ERROR(log, getExceptionMessage(exception, true)); + LOG_ERROR(log, fmt::runtime(getExceptionMessage(exception, true))); if (responder && !responder_finished) { @@ -1399,6 +1451,29 @@ namespace return false; } + void Call::addQueryDetailsToResult() + { + *result.mutable_query_id() = query_context->getClientInfo().current_query_id; + *result.mutable_time_zone() = DateLUT::instance().getTimeZone(); + } + + void Call::addOutputFormatToResult() + { + *result.mutable_output_format() = output_format; + } + + void Call::addOutputColumnsNamesAndTypesToResult(const Block & header) + { + if (!send_output_columns_names_and_types) + return; + for (const auto & column : header) + { + auto & name_and_type = *result.add_output_columns(); + *name_and_type.mutable_name() = column.name; + *name_and_type.mutable_type() = column.type->getName(); + } + } + void Call::addProgressToResult() { auto values = progress.fetchAndResetPiecewiseAtomically(); @@ -1419,10 +1494,10 @@ namespace return; PODArray memory; - if (compression_method != CompressionMethod::None) + if (output_compression_method != CompressionMethod::None) memory.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data. std::unique_ptr buf = std::make_unique>>(memory); - buf = wrapWriteBufferWithCompressionMethod(std::move(buf), compression_method, compression_level); + buf = wrapWriteBufferWithCompressionMethod(std::move(buf), output_compression_method, output_compression_level); auto format = query_context->getOutputFormat(output_format, *buf, totals); format->write(materializeBlock(totals)); format->finalize(); @@ -1437,10 +1512,10 @@ namespace return; PODArray memory; - if (compression_method != CompressionMethod::None) + if (output_compression_method != CompressionMethod::None) memory.resize(DBMS_DEFAULT_BUFFER_SIZE); /// Must have enough space for compressed data. std::unique_ptr buf = std::make_unique>>(memory); - buf = wrapWriteBufferWithCompressionMethod(std::move(buf), compression_method, compression_level); + buf = wrapWriteBufferWithCompressionMethod(std::move(buf), output_compression_method, output_compression_level); auto format = query_context->getOutputFormat(output_format, *buf, extremes); format->write(materializeBlock(extremes)); format->finalize(); @@ -1777,8 +1852,9 @@ void GRPCServer::start() builder.RegisterService(&grpc_service); builder.SetMaxSendMessageSize(iserver.config().getInt("grpc.max_send_message_size", -1)); builder.SetMaxReceiveMessageSize(iserver.config().getInt("grpc.max_receive_message_size", -1)); - builder.SetDefaultCompressionAlgorithm(parseCompressionAlgorithm(iserver.config().getString("grpc.compression", "none"))); - builder.SetDefaultCompressionLevel(parseCompressionLevel(iserver.config().getString("grpc.compression_level", "none"))); + auto default_transport_compression = TransportCompression::fromConfiguration(iserver.config()); + builder.SetDefaultCompressionAlgorithm(default_transport_compression.algorithm); + builder.SetDefaultCompressionLevel(default_transport_compression.level); queue = builder.AddCompletionQueue(); grpc_server = builder.BuildAndStart(); diff --git a/src/Server/HTTP/HTTPServerConnection.cpp b/src/Server/HTTP/HTTPServerConnection.cpp index 7020b8e9a23..e365c9f31d0 100644 --- a/src/Server/HTTP/HTTPServerConnection.cpp +++ b/src/Server/HTTP/HTTPServerConnection.cpp @@ -36,7 +36,7 @@ void HTTPServerConnection::run() if (request.isSecure()) { - size_t hsts_max_age = context->getSettings().hsts_max_age.value; + size_t hsts_max_age = context->getSettingsRef().hsts_max_age.value; if (hsts_max_age > 0) response.add("Strict-Transport-Security", "max-age=" + std::to_string(hsts_max_age)); diff --git a/src/Server/HTTPHandler.cpp b/src/Server/HTTPHandler.cpp index 5253e66be92..a42df54aed7 100644 --- a/src/Server/HTTPHandler.cpp +++ b/src/Server/HTTPHandler.cpp @@ -915,7 +915,10 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse } processQuery(request, params, response, used_output, query_scope); - LOG_DEBUG(log, (request_credentials ? "Authentication in progress..." : "Done processing query")); + if (request_credentials) + LOG_DEBUG(log, "Authentication in progress..."); + else + LOG_DEBUG(log, "Done processing query"); } catch (...) { diff --git a/src/Server/InterserverIOHTTPHandler.cpp b/src/Server/InterserverIOHTTPHandler.cpp index 082f7cc2e33..9506c5c133f 100644 --- a/src/Server/InterserverIOHTTPHandler.cpp +++ b/src/Server/InterserverIOHTTPHandler.cpp @@ -138,9 +138,9 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe write_response(message); if (is_real_error) - LOG_ERROR(log, message); + LOG_ERROR(log, fmt::runtime(message)); else - LOG_INFO(log, message); + LOG_INFO(log, fmt::runtime(message)); } catch (...) { @@ -148,7 +148,7 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe std::string message = getCurrentExceptionMessage(false); write_response(message); - LOG_ERROR(log, message); + LOG_ERROR(log, fmt::runtime(message)); } } diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 9808b538280..04e43ed63aa 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -105,7 +105,7 @@ void PostgreSQLHandler::run() "0A000", "Command is not supported"), true); - LOG_ERROR(log, Poco::format("Command is not supported. Command code %d", static_cast(message_type))); + LOG_ERROR(log, "Command is not supported. Command code {:d}", static_cast(message_type)); message_transport->dropMessage(); } } @@ -222,7 +222,7 @@ void PostgreSQLHandler::cancelRequest() std::unique_ptr msg = message_transport->receiveWithPayloadSize(8); - String query = Poco::format("KILL QUERY WHERE query_id = 'postgres:%d:%d'", msg->process_id, msg->secret_key); + String query = fmt::format("KILL QUERY WHERE query_id = 'postgres:{:d}:{:d}'", msg->process_id, msg->secret_key); ReadBufferFromString replacement(query); auto query_context = session->makeQueryContext(); @@ -287,7 +287,7 @@ void PostgreSQLHandler::processQuery() { secret_key = dis(gen); auto query_context = session->makeQueryContext(); - query_context->setCurrentQueryId(Poco::format("postgres:%d:%d", connection_id, secret_key)); + query_context->setCurrentQueryId(fmt::format("postgres:{:d}:{:d}", connection_id, secret_key)); CurrentThread::QueryScope query_scope{query_context}; ReadBufferFromString read_buf(spl_query); diff --git a/src/Server/TCPHandler.cpp b/src/Server/TCPHandler.cpp index 6fa2b25d181..99523ff09e3 100644 --- a/src/Server/TCPHandler.cpp +++ b/src/Server/TCPHandler.cpp @@ -359,6 +359,7 @@ void TCPHandler::runImpl() return true; sendProgress(); + sendProfileEvents(); sendLogs(); return false; @@ -466,7 +467,7 @@ void TCPHandler::runImpl() } const auto & e = *exception; - LOG_ERROR(log, getExceptionMessage(e, true)); + LOG_ERROR(log, fmt::runtime(getExceptionMessage(e, true))); sendException(*exception, send_exception_with_stack_trace); } } @@ -1392,6 +1393,12 @@ void TCPHandler::receiveQuery() if (is_interserver_mode) { ClientInfo original_session_client_info = session->getClientInfo(); + + /// Cleanup fields that should not be reused from previous query. + original_session_client_info.current_user.clear(); + original_session_client_info.current_query_id.clear(); + original_session_client_info.current_address = {}; + session = std::make_unique(server.context(), ClientInfo::Interface::TCP_INTERSERVER); session->getClientInfo() = original_session_client_info; } diff --git a/src/Server/grpc_protos/clickhouse_grpc.proto b/src/Server/grpc_protos/clickhouse_grpc.proto index c86c74535c5..4593cfff096 100644 --- a/src/Server/grpc_protos/clickhouse_grpc.proto +++ b/src/Server/grpc_protos/clickhouse_grpc.proto @@ -45,21 +45,19 @@ message ExternalTable { map settings = 5; } -enum CompressionAlgorithm { - NO_COMPRESSION = 0; - DEFLATE = 1; - GZIP = 2; - STREAM_GZIP = 3; -} - -enum CompressionLevel { - COMPRESSION_NONE = 0; - COMPRESSION_LOW = 1; - COMPRESSION_MEDIUM = 2; - COMPRESSION_HIGH = 3; -} - -message Compression { +message ObsoleteTransportCompression { + enum CompressionAlgorithm { + NO_COMPRESSION = 0; + DEFLATE = 1; + GZIP = 2; + STREAM_GZIP = 3; + } + enum CompressionLevel { + COMPRESSION_NONE = 0; + COMPRESSION_LOW = 1; + COMPRESSION_MEDIUM = 2; + COMPRESSION_HIGH = 3; + } CompressionAlgorithm algorithm = 1; CompressionLevel level = 2; } @@ -84,6 +82,9 @@ message QueryInfo { // Default output format. If not specified, 'TabSeparated' is used. string output_format = 7; + // Set it if you want the names and the types of output columns to be sent to the client. + bool send_output_columns = 24; + repeated ExternalTable external_tables = 8; string user_name = 9; @@ -102,16 +103,16 @@ message QueryInfo { // `next_query_info` is allowed to be set only if a method with streaming input (i.e. ExecuteQueryWithStreamInput() or ExecuteQueryWithStreamIO()) is used. bool next_query_info = 16; - /// Controls how a ClickHouse server will compress query execution results before sending back to the client. - /// If not set the compression settings from the configuration file will be used. - Compression result_compression = 17; - - // Compression type for `input_data`, `output_data`, `totals` and `extremes`. + // Compression type for `input_data`. // Supported compression types: none, gzip(gz), deflate, brotli(br), lzma(xz), zstd(zst), lz4, bz2. - // When used for `input_data` the client is responsible to compress data before putting it into `input_data`. - // When used for `output_data` or `totals` or `extremes` the client receives compressed data and should decompress it by itself. - // In the latter case consider to specify also `compression_level`. - string compression_type = 18; + // The client is responsible to compress data before putting it into `input_data`. + string input_compression_type = 20; + + // Compression type for `output_data`, `totals` and `extremes`. + // Supported compression types: none, gzip(gz), deflate, brotli(br), lzma(xz), zstd(zst), lz4, bz2. + // The client receives compressed data and should decompress it by itself. + // Consider also setting `output_compression_level`. + string output_compression_type = 21; // Compression level. // WARNING: If it's not specified the compression level is set to zero by default which might be not the best choice for some compression types (see below). @@ -123,7 +124,23 @@ message QueryInfo { // zstd: 1..22; 3 is recommended by default (compression level 0 also means 3) // lz4: 0..16; values < 0 mean fast acceleration // bz2: 1..9 - int32 compression_level = 19; + int32 output_compression_level = 19; + + // Transport compression is an alternative way to make the server to compress its response. + // This kind of compression implies that instead of compressing just `output` the server will compress whole packed messages of the `Result` type, + // and then gRPC implementation on client side will decompress those messages so client code won't be bothered with decompression. + // Here is a big difference between the transport compression and the compression enabled by setting `output_compression_type` because + // in case of the transport compression the client code receives already decompressed data in `output`. + // If the transport compression is not set here it can still be enabled by the server configuration. + // Supported compression types: none, deflate, gzip, stream_gzip + // Supported compression levels: 0..3 + // WARNING: Don't set `transport_compression` and `output_compression` at the same time because it will make the server to compress its output twice! + string transport_compression_type = 22; + int32 transport_compression_level = 23; + + /// Obsolete fields, should not be used in new code. + ObsoleteTransportCompression obsolete_result_compression = 17; + string obsolete_compression_type = 18; } enum LogsLevel { @@ -173,7 +190,17 @@ message Exception { // Result of execution of a query which is sent back by the ClickHouse server to the client. message Result { - // Output of the query, represented in the `output_format` or in a format specified in `query`. + string query_id = 9; + string time_zone = 10; + + // The format in which `output`, `totals` and `extremes` are written. + // It's either the same as `output_format` specified in `QueryInfo` or the format specified in the query itself. + string output_format = 11; + + // The names and types of columns of the result written in `output`. + repeated NameAndType output_columns = 12; + + // Output of the query, represented in the `output_format`. bytes output = 1; bytes totals = 2; bytes extremes = 3; diff --git a/src/Storages/Distributed/DirectoryMonitor.cpp b/src/Storages/Distributed/DirectoryMonitor.cpp index 0c41cf71386..d7422b1ddbc 100644 --- a/src/Storages/Distributed/DirectoryMonitor.cpp +++ b/src/Storages/Distributed/DirectoryMonitor.cpp @@ -132,6 +132,7 @@ namespace struct DistributedHeader { + UInt64 revision = 0; Settings insert_settings; std::string insert_query; ClientInfo client_info; @@ -166,9 +167,8 @@ namespace /// Read the parts of the header. ReadBufferFromString header_buf(header_data); - UInt64 initiator_revision; - readVarUInt(initiator_revision, header_buf); - if (DBMS_TCP_PROTOCOL_VERSION < initiator_revision) + readVarUInt(distributed_header.revision, header_buf); + if (DBMS_TCP_PROTOCOL_VERSION < distributed_header.revision) { LOG_WARNING(log, "ClickHouse shard version is older than ClickHouse initiator version. It may lack support for new features."); } @@ -177,7 +177,7 @@ namespace distributed_header.insert_settings.read(header_buf); if (header_buf.hasPendingData()) - distributed_header.client_info.read(header_buf, initiator_revision); + distributed_header.client_info.read(header_buf, distributed_header.revision); if (header_buf.hasPendingData()) { @@ -188,10 +188,12 @@ namespace if (header_buf.hasPendingData()) { - NativeReader header_block_in(header_buf, DBMS_TCP_PROTOCOL_VERSION); + NativeReader header_block_in(header_buf, distributed_header.revision); distributed_header.block_header = header_block_in.read(); if (!distributed_header.block_header) - throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read header from the {} batch", in.getFileName()); + throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, + "Cannot read header from the {} batch. Data was written with protocol version {}, current version: {}", + in.getFileName(), distributed_header.revision, DBMS_TCP_PROTOCOL_VERSION); } /// Add handling new data here, for example: @@ -264,10 +266,10 @@ namespace return nullptr; } - void writeAndConvert(RemoteInserter & remote, ReadBufferFromFile & in) + void writeAndConvert(RemoteInserter & remote, const DistributedHeader & distributed_header, ReadBufferFromFile & in) { CompressedReadBuffer decompressing_in(in); - NativeReader block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION); + NativeReader block_in(decompressing_in, distributed_header.revision); while (Block block = block_in.read()) { @@ -304,7 +306,7 @@ namespace { LOG_TRACE(log, "Processing batch {} with old format (no header)", in.getFileName()); - writeAndConvert(remote, in); + writeAndConvert(remote, distributed_header, in); return; } @@ -314,14 +316,20 @@ namespace "Structure does not match (remote: {}, local: {}), implicit conversion will be done", remote.getHeader().dumpStructure(), distributed_header.block_header.dumpStructure()); - writeAndConvert(remote, in); + writeAndConvert(remote, distributed_header, in); return; } /// If connection does not use compression, we have to uncompress the data. if (!compression_expected) { - writeAndConvert(remote, in); + writeAndConvert(remote, distributed_header, in); + return; + } + + if (distributed_header.revision != remote.getServerRevision()) + { + writeAndConvert(remote, distributed_header, in); return; } @@ -915,10 +923,10 @@ public: { in = std::make_unique(file_name); decompressing_in = std::make_unique(*in); - block_in = std::make_unique(*decompressing_in, DBMS_TCP_PROTOCOL_VERSION); log = &Poco::Logger::get("DirectoryMonitorSource"); - readDistributedHeader(*in, log); + auto distributed_header = readDistributedHeader(*in, log); + block_in = std::make_unique(*decompressing_in, distributed_header.revision); first_block = block_in->read(); } @@ -1040,7 +1048,7 @@ void StorageDistributedDirectoryMonitor::processFilesWithBatching(const std::map LOG_DEBUG(log, "Processing batch {} with old format (no header/rows)", in.getFileName()); CompressedReadBuffer decompressing_in(in); - NativeReader block_in(decompressing_in, DBMS_TCP_PROTOCOL_VERSION); + NativeReader block_in(decompressing_in, distributed_header.revision); while (Block block = block_in.read()) { diff --git a/src/Storages/FileLog/StorageFileLog.cpp b/src/Storages/FileLog/StorageFileLog.cpp index 1ae8a5bb22d..f89caaec685 100644 --- a/src/Storages/FileLog/StorageFileLog.cpp +++ b/src/Storages/FileLog/StorageFileLog.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -966,7 +967,9 @@ bool StorageFileLog::updateFileInfos() NamesAndTypesList StorageFileLog::getVirtuals() const { - return NamesAndTypesList{{"_filename", std::make_shared()}, {"_offset", std::make_shared()}}; + return NamesAndTypesList{ + {"_filename", std::make_shared(std::make_shared())}, + {"_offset", std::make_shared()}}; } Names StorageFileLog::getVirtualColumnNames() diff --git a/src/Storages/HDFS/ReadBufferFromHDFS.cpp b/src/Storages/HDFS/ReadBufferFromHDFS.cpp index 15f6c73bc51..0ad55162fb2 100644 --- a/src/Storages/HDFS/ReadBufferFromHDFS.cpp +++ b/src/Storages/HDFS/ReadBufferFromHDFS.cpp @@ -173,7 +173,7 @@ off_t ReadBufferFromHDFS::seek(off_t offset_, int whence) return getPosition(); } - pos = working_buffer.end(); + resetWorkingBuffer(); impl->seek(offset_, whence); return impl->getPosition(); } diff --git a/src/Storages/HDFS/StorageHDFS.cpp b/src/Storages/HDFS/StorageHDFS.cpp index 95cc4542977..7b07e929c76 100644 --- a/src/Storages/HDFS/StorageHDFS.cpp +++ b/src/Storages/HDFS/StorageHDFS.cpp @@ -258,9 +258,15 @@ Block HDFSSource::getHeader(const StorageMetadataPtr & metadata_snapshot, bool n auto header = metadata_snapshot->getSampleBlock(); /// Note: AddingDefaultsBlockInputStream doesn't change header. if (need_path_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_path"}); if (need_file_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_file"}); return header; } @@ -320,6 +326,7 @@ HDFSSource::HDFSSource( void HDFSSource::onCancel() { + std::lock_guard lock(reader_mutex); if (reader) reader->cancel(); } @@ -365,41 +372,47 @@ String HDFSSource::getName() const Chunk HDFSSource::generate() { - if (!reader) - return {}; - - Chunk chunk; - if (reader->pull(chunk)) + while (true) { - Columns columns = chunk.getColumns(); - UInt64 num_rows = chunk.getNumRows(); + if (!reader || isCancelled()) + break; - /// Enrich with virtual columns. - if (need_path_column) + Chunk chunk; + if (reader->pull(chunk)) { - auto column = DataTypeString().createColumnConst(num_rows, current_path); - columns.push_back(column->convertToFullColumnIfConst()); + Columns columns = chunk.getColumns(); + UInt64 num_rows = chunk.getNumRows(); + + /// Enrich with virtual columns. + if (need_path_column) + { + auto column = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, current_path); + columns.push_back(column->convertToFullColumnIfConst()); + } + + if (need_file_column) + { + size_t last_slash_pos = current_path.find_last_of('/'); + auto file_name = current_path.substr(last_slash_pos + 1); + + auto column = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, std::move(file_name)); + columns.push_back(column->convertToFullColumnIfConst()); + } + + return Chunk(std::move(columns), num_rows); } - if (need_file_column) { - size_t last_slash_pos = current_path.find_last_of('/'); - auto file_name = current_path.substr(last_slash_pos + 1); + std::lock_guard lock(reader_mutex); + reader.reset(); + pipeline.reset(); + read_buf.reset(); - auto column = DataTypeString().createColumnConst(num_rows, std::move(file_name)); - columns.push_back(column->convertToFullColumnIfConst()); + if (!initialize()) + break; } - - return Chunk(std::move(columns), num_rows); } - - reader.reset(); - pipeline.reset(); - read_buf.reset(); - - if (!initialize()) - return {}; - return generate(); + return {}; } @@ -685,9 +698,8 @@ void registerStorageHDFS(StorageFactory & factory) NamesAndTypesList StorageHDFS::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } } diff --git a/src/Storages/HDFS/StorageHDFS.h b/src/Storages/HDFS/StorageHDFS.h index 8e455189bc6..0e23e45f4c5 100644 --- a/src/Storages/HDFS/StorageHDFS.h +++ b/src/Storages/HDFS/StorageHDFS.h @@ -150,6 +150,8 @@ private: std::unique_ptr read_buf; std::unique_ptr pipeline; std::unique_ptr reader; + /// onCancel and generate can be called concurrently + std::mutex reader_mutex; String current_path; /// Recreate ReadBuffer and PullingPipelineExecutor for each file. diff --git a/src/Storages/HDFS/StorageHDFSCluster.cpp b/src/Storages/HDFS/StorageHDFSCluster.cpp index ba1cc045fbf..dde3329040b 100644 --- a/src/Storages/HDFS/StorageHDFSCluster.cpp +++ b/src/Storages/HDFS/StorageHDFSCluster.cpp @@ -2,17 +2,9 @@ #if USE_HDFS -#include -#include #include #include -#include -#include -#include #include -#include -#include -#include #include #include #include @@ -21,7 +13,6 @@ #include #include #include -#include #include #include #include @@ -29,16 +20,13 @@ #include #include #include -#include -#include #include -#include -#include -#include + namespace DB { + StorageHDFSCluster::StorageHDFSCluster( String cluster_name_, const String & uri_, @@ -69,7 +57,7 @@ Pipe StorageHDFSCluster::read( size_t /*max_block_size*/, unsigned /*num_streams*/) { - auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettings()); + auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettingsRef()); auto iterator = std::make_shared(context, uri); auto callback = std::make_shared([iterator]() mutable -> String @@ -138,9 +126,8 @@ QueryProcessingStage::Enum StorageHDFSCluster::getQueryProcessingStage( NamesAndTypesList StorageHDFSCluster::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } diff --git a/src/Storages/HDFS/WriteBufferFromHDFS.cpp b/src/Storages/HDFS/WriteBufferFromHDFS.cpp index 2addfc0069f..42ec3962beb 100644 --- a/src/Storages/HDFS/WriteBufferFromHDFS.cpp +++ b/src/Storages/HDFS/WriteBufferFromHDFS.cpp @@ -25,7 +25,7 @@ struct WriteBufferFromHDFS::WriteBufferFromHDFSImpl HDFSBuilderWrapper builder; HDFSFSPtr fs; - explicit WriteBufferFromHDFSImpl( + WriteBufferFromHDFSImpl( const std::string & hdfs_uri_, const Poco::Util::AbstractConfiguration & config_, int replication_, diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index af357e13ca7..3040ad23283 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -76,9 +76,15 @@ public: static Block getHeader(Block header, const SourcesInfoPtr & source_info) { if (source_info->need_path_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_path"}); if (source_info->need_file_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_file"}); return header; } @@ -87,9 +93,9 @@ public: { ColumnsDescription columns_description{header.getNamesAndTypesList()}; if (source_info->need_path_column) - columns_description.add({"_path", std::make_shared()}); + columns_description.add({"_path", std::make_shared(std::make_shared())}); if (source_info->need_file_column) - columns_description.add({"_file", std::make_shared()}); + columns_description.add({"_file", std::make_shared(std::make_shared())}); return columns_description; } @@ -211,7 +217,7 @@ public: /// Enrich with virtual columns. if (source_info->need_path_column) { - auto column = DataTypeString().createColumnConst(num_rows, current_path); + auto column = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, current_path); columns.push_back(column->convertToFullColumnIfConst()); } @@ -220,7 +226,8 @@ public: size_t last_slash_pos = current_path.find_last_of('/'); auto file_name = current_path.substr(last_slash_pos + 1); - auto column = DataTypeString().createColumnConst(num_rows, std::move(file_name)); + auto column + = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, std::move(file_name)); columns.push_back(column->convertToFullColumnIfConst()); } return Chunk(std::move(columns), num_rows); @@ -633,7 +640,9 @@ SinkToStoragePtr StorageHive::write(const ASTPtr & /*query*/, const StorageMetad NamesAndTypesList StorageHive::getVirtuals() const { - return NamesAndTypesList{{"_path", std::make_shared()}, {"_file", std::make_shared()}}; + return NamesAndTypesList{ + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } void registerStorageHive(StorageFactory & factory) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 6342c3f6b47..fc79c7d174f 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -217,7 +217,6 @@ public: /// Extract data from the backup and put it to the storage. virtual RestoreDataTasks restoreFromBackup(const BackupPtr & backup, const String & data_path_in_backup, const ASTs & partitions, ContextMutablePtr context); -protected: /// Returns whether the column is virtual - by default all columns are real. /// Initially reserved virtual column name may be shadowed by real column. bool isVirtualColumn(const String & column_name, const StorageMetadataPtr & metadata_snapshot) const; diff --git a/src/Storages/Kafka/StorageKafka.cpp b/src/Storages/Kafka/StorageKafka.cpp index 6101eb04af6..30acbcdf62b 100644 --- a/src/Storages/Kafka/StorageKafka.cpp +++ b/src/Storages/Kafka/StorageKafka.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -16,13 +17,15 @@ #include #include #include +#include #include #include #include #include -#include #include #include +#include +#include #include #include #include @@ -36,8 +39,6 @@ #include #include #include -#include -#include namespace DB @@ -807,15 +808,14 @@ void registerStorageKafka(StorageFactory & factory) NamesAndTypesList StorageKafka::getVirtuals() const { auto result = NamesAndTypesList{ - {"_topic", std::make_shared()}, + {"_topic", std::make_shared(std::make_shared())}, {"_key", std::make_shared()}, {"_offset", std::make_shared()}, {"_partition", std::make_shared()}, {"_timestamp", std::make_shared(std::make_shared())}, {"_timestamp_ms", std::make_shared(std::make_shared(3))}, {"_headers.name", std::make_shared(std::make_shared())}, - {"_headers.value", std::make_shared(std::make_shared())} - }; + {"_headers.value", std::make_shared(std::make_shared())}}; if (kafka_settings->kafka_handle_error_mode == HandleKafkaErrorMode::STREAM) { result.push_back({"_raw_message", std::make_shared()}); diff --git a/src/Storages/KeyDescription.cpp b/src/Storages/KeyDescription.cpp index 6a2f4bbb055..f100f129cda 100644 --- a/src/Storages/KeyDescription.cpp +++ b/src/Storages/KeyDescription.cpp @@ -8,6 +8,9 @@ #include #include #include +#include +#include +#include namespace DB @@ -161,4 +164,17 @@ KeyDescription KeyDescription::buildEmptyKey() return result; } +KeyDescription KeyDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr context) +{ + KeyDescription result; + if (str.empty()) + return result; + + ParserExpression parser; + ASTPtr ast = parseQuery(parser, "(" + str + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + FunctionNameNormalizer().visit(ast.get()); + + return getKeyFromAST(ast, columns, context); +} + } diff --git a/src/Storages/KeyDescription.h b/src/Storages/KeyDescription.h index 81803c3e44b..527a36124aa 100644 --- a/src/Storages/KeyDescription.h +++ b/src/Storages/KeyDescription.h @@ -76,6 +76,9 @@ struct KeyDescription /// Substitute modulo with moduloLegacy. Used in KeyCondition to allow proper comparison with keys. static bool moduloToModuloLegacyRecursive(ASTPtr node_expr); + + /// Parse description from string + static KeyDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context); }; } diff --git a/src/Storages/MergeTree/DataPartsExchange.cpp b/src/Storages/MergeTree/DataPartsExchange.cpp index 2a964aecd4e..19d990d7c2d 100644 --- a/src/Storages/MergeTree/DataPartsExchange.cpp +++ b/src/Storages/MergeTree/DataPartsExchange.cpp @@ -19,6 +19,8 @@ #include #include #include +#include + namespace fs = std::filesystem; @@ -220,7 +222,7 @@ void Service::sendPartFromMemory( auto projection_sample_block = metadata_snapshot->projections.get(name).sample_block; auto part_in_memory = asInMemoryPart(projection); if (!part_in_memory) - throw Exception("Projection " + name + " of part " + part->name + " is not stored in memory", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Projection {} of part {} is not stored in memory", name, part->name); writeStringBinary(name, out); projection->checksums.write(out); @@ -230,7 +232,7 @@ void Service::sendPartFromMemory( auto part_in_memory = asInMemoryPart(part); if (!part_in_memory) - throw Exception("Part " + part->name + " is not stored in memory", ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} is not stored in memory", part->name); NativeWriter block_out(out, 0, metadata_snapshot->getSampleBlock()); part->checksums.write(out); @@ -298,7 +300,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); if (hashing_out.count() != size) - throw Exception("Unexpected size of file " + path, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}, expected {} got {}", path, hashing_out.count(), size); writePODBinary(hashing_out.getHash(), out); @@ -312,6 +314,10 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk( void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part, WriteBuffer & out) { + auto disk = part->volume->getDisk(); + if (!disk->supportZeroCopyReplication()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Disk '{}' doesn't support zero-copy replication", disk->getName()); + /// We'll take a list of files from the list of checksums. MergeTreeData::DataPart::Checksums checksums = part->checksums; /// Add files that are not in the checksum list. @@ -319,11 +325,13 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part for (const auto & file_name : file_names_without_checksums) checksums.files[file_name] = {}; - auto disk = part->volume->getDisk(); - if (!disk->supportZeroCopyReplication()) - throw Exception(fmt::format("disk {} doesn't support zero-copy replication", disk->getName()), ErrorCodes::LOGICAL_ERROR); + std::vector paths; + paths.reserve(checksums.files.size()); + for (const auto & it : checksums.files) + paths.push_back(fs::path(part->getFullRelativePath()) / it.first); - part->storage.lockSharedData(*part); + /// Serialized metadatadatas with zero ref counts. + auto metadatas = disk->getSerializedMetadata(paths); String part_id = part->getUniqueId(); writeStringBinary(part_id, out); @@ -331,29 +339,32 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part writeBinary(checksums.files.size(), out); for (const auto & it : checksums.files) { - String file_name = it.first; - - String metadata_file = fs::path(disk->getPath()) / part->getFullRelativePath() / file_name; - - fs::path metadata(metadata_file); + const String & file_name = it.first; + String file_path_prefix = fs::path(part->getFullRelativePath()) / file_name; + /// Just some additional checks + String metadata_file_path = fs::path(disk->getPath()) / file_path_prefix; + fs::path metadata(metadata_file_path); if (!fs::exists(metadata)) - throw Exception("Remote metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA); + throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not exists", file_name); if (!fs::is_regular_file(metadata)) - throw Exception("Remote metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA); - UInt64 file_size = fs::file_size(metadata); + throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not a file", file_name); + + /// Actual metadata send + auto metadata_str = metadatas[file_path_prefix]; + UInt64 file_size = metadata_str.size(); + ReadBufferFromString buf(metadata_str); writeStringBinary(it.first, out); writeBinary(file_size, out); - auto file_in = createReadBufferFromFileBase(metadata_file, /* settings= */ {}); HashingWriteBuffer hashing_out(out); - copyDataWithThrottler(*file_in, hashing_out, blocker.getCounter(), data.getSendsThrottler()); + copyDataWithThrottler(buf, hashing_out, blocker.getCounter(), data.getSendsThrottler()); if (blocker.isCancelled()) throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED); if (hashing_out.count() != file_size) - throw Exception("Unexpected size of file " + metadata_file, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART); + throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}", metadata_file_path); writePODBinary(hashing_out.getHash(), out); } @@ -368,7 +379,7 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name) if (part) return part; - throw Exception("No part " + name + " in table", ErrorCodes::NO_SUCH_DATA_PART); + throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", name); } MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( @@ -425,7 +436,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( } if (!capability.empty()) { - std::sort(capability.begin(), capability.end()); + ::sort(capability.begin(), capability.end()); capability.erase(std::unique(capability.begin(), capability.end()), capability.end()); const String & remote_fs_metadata = boost::algorithm::join(capability, ", "); uri.addQueryParameter("remote_fs_metadata", remote_fs_metadata); @@ -509,9 +520,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( if (!try_zero_copy) throw Exception("Got unexpected 'remote_fs_metadata' cookie", ErrorCodes::LOGICAL_ERROR); if (std::find(capability.begin(), capability.end(), remote_fs_metadata) == capability.end()) - throw Exception(fmt::format("Got 'remote_fs_metadata' cookie {}, expect one from {}", remote_fs_metadata, fmt::join(capability, ", ")), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got 'remote_fs_metadata' cookie {}, expect one from {}", remote_fs_metadata, fmt::join(capability, ", ")); if (server_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY) - throw Exception(fmt::format("Got 'remote_fs_metadata' cookie with old protocol version {}", server_protocol_version), ErrorCodes::LOGICAL_ERROR); + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got 'remote_fs_metadata' cookie with old protocol version {}", server_protocol_version); if (part_type == "InMemory") throw Exception("Got 'remote_fs_metadata' cookie for in-memory part", ErrorCodes::INCORRECT_PART_TYPE); @@ -523,7 +534,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart( { if (e.code() != ErrorCodes::S3_ERROR && e.code() != ErrorCodes::ZERO_COPY_REPLICATION_ERROR) throw; - LOG_WARNING(log, e.message() + " Will retry fetching part without zero-copy."); + LOG_WARNING(log, fmt::runtime(e.message() + " Will retry fetching part without zero-copy.")); /// Try again but without zero-copy return fetchPart(metadata_snapshot, context, part_name, replica_path, host, port, timeouts, user, password, interserver_scheme, throttler, to_detached, tmp_prefix_, nullptr, false, disk); @@ -593,7 +604,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( CompressionCodecFactory::instance().get("NONE", {})); part_out.write(block); - part_out.writeSuffixAndFinalizePart(new_projection_part); + part_out.finalizePart(new_projection_part, false); new_projection_part->checksums.checkEqual(checksums, /* have_uncompressed = */ true); new_data_part->addProjectionPart(projection_name, std::move(new_projection_part)); } @@ -617,7 +628,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToMemory( CompressionCodecFactory::instance().get("NONE", {})); part_out.write(block); - part_out.writeSuffixAndFinalizePart(new_data_part); + part_out.finalizePart(new_data_part, false); new_data_part->checksums.checkEqual(checksums, /* have_uncompressed = */ true); return new_data_part; @@ -647,9 +658,10 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( /// Otherwise malicious ClickHouse replica may force us to write to arbitrary path. String absolute_file_path = fs::weakly_canonical(fs::path(part_download_path) / file_name); if (!startsWith(absolute_file_path, fs::weakly_canonical(part_download_path).string())) - throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + part_download_path + ")." - " This may happen if we are trying to download part from malicious replica or logical error.", - ErrorCodes::INSECURE_PATH); + throw Exception(ErrorCodes::INSECURE_PATH, + "File path ({}) doesn't appear to be inside part path ({}). " + "This may happen if we are trying to download part from malicious replica or logical error.", + absolute_file_path, part_download_path); auto file_out = disk->writeFile(fs::path(part_download_path) / file_name); HashingWriteBuffer hashing_out(*file_out); @@ -668,8 +680,10 @@ void Fetcher::downloadBaseOrProjectionPartToDisk( readPODBinary(expected_hash, in); if (expected_hash != hashing_out.getHash()) - throw Exception("Checksum mismatch for file " + fullPath(disk, (fs::path(part_download_path) / file_name).string()) + " transferred from " + replica_path, - ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, + "Checksum mismatch for file {} transferred from {}", + fullPath(disk, (fs::path(part_download_path) / file_name).string()), + replica_path); if (file_name != "checksums.txt" && file_name != "columns.txt" && @@ -760,11 +774,14 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( if (!disk->supportZeroCopyReplication() || !disk->checkUniqueId(part_id)) { - throw Exception(fmt::format("Part {} unique id {} doesn't exist on {}.", part_name, part_id, disk->getName()), ErrorCodes::ZERO_COPY_REPLICATION_ERROR); + throw Exception(ErrorCodes::ZERO_COPY_REPLICATION_ERROR, "Part {} unique id {} doesn't exist on {}.", part_name, part_id, disk->getName()); } + LOG_DEBUG(log, "Downloading Part {} unique id {} metadata onto disk {}.", part_name, part_id, disk->getName()); + data.lockSharedDataTemporary(part_name, part_id, disk); + static const String TMP_PREFIX = "tmp-fetch_"; String tmp_prefix = tmp_prefix_.empty() ? TMP_PREFIX : tmp_prefix_; @@ -772,7 +789,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( String part_download_path = fs::path(data.getRelativeDataPath()) / part_relative_path / ""; if (disk->exists(part_download_path)) - throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS); + throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Directory {} already exists.", fullPath(disk, part_download_path)); CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch}; @@ -815,8 +832,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( if (expected_hash != hashing_out.getHash()) { - throw Exception("Checksum mismatch for file " + metadata_file + " transferred from " + replica_path, - ErrorCodes::CHECKSUM_DOESNT_MATCH); + throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH, + "Checksum mismatch for file {} transferred from {}", + metadata_file, replica_path); } } } @@ -828,7 +846,10 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta( new_data_part->modification_time = time(nullptr); new_data_part->loadColumnsChecksumsIndexes(true, false); - new_data_part->storage.lockSharedData(*new_data_part); + data.lockSharedData(*new_data_part, /* replace_existing_lock = */ true); + + LOG_DEBUG(log, "Download of part {} unique id {} metadata onto disk {} finished.", + part_name, part_id, disk->getName()); return new_data_part; } diff --git a/src/Storages/MergeTree/DataPartsExchange.h b/src/Storages/MergeTree/DataPartsExchange.h index 0c12cc51cc7..8dfcfef9a8b 100644 --- a/src/Storages/MergeTree/DataPartsExchange.h +++ b/src/Storages/MergeTree/DataPartsExchange.h @@ -63,7 +63,7 @@ private: class Fetcher final : private boost::noncopyable { public: - explicit Fetcher(MergeTreeData & data_) : data(data_), log(&Poco::Logger::get("Fetcher")) {} + explicit Fetcher(StorageReplicatedMergeTree & data_) : data(data_), log(&Poco::Logger::get("Fetcher")) {} /// Downloads a part to tmp_directory. If to_detached - downloads to the `detached` directory. MergeTreeData::MutableDataPartPtr fetchPart( @@ -129,7 +129,7 @@ private: PooledReadWriteBufferFromHTTP & in, ThrottlerPtr throttler); - MergeTreeData & data; + StorageReplicatedMergeTree & data; Poco::Logger * log; }; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.cpp b/src/Storages/MergeTree/IMergeTreeDataPart.cpp index 44358e7a17a..9028023dc80 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.cpp +++ b/src/Storages/MergeTree/IMergeTreeDataPart.cpp @@ -1,6 +1,7 @@ #include "IMergeTreeDataPart.h" #include +#include #include #include #include @@ -98,7 +99,7 @@ void IMergeTreeDataPart::MinMaxIndex::load(const MergeTreeData & data, const Dis initialized = true; } -void IMergeTreeDataPart::MinMaxIndex::store( +IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::store( const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & out_checksums) const { auto metadata_snapshot = data.getInMemoryMetadataPtr(); @@ -107,10 +108,10 @@ void IMergeTreeDataPart::MinMaxIndex::store( auto minmax_column_names = data.getMinMaxColumnsNames(partition_key); auto minmax_column_types = data.getMinMaxColumnsTypes(partition_key); - store(minmax_column_names, minmax_column_types, disk_, part_path, out_checksums); + return store(minmax_column_names, minmax_column_types, disk_, part_path, out_checksums); } -void IMergeTreeDataPart::MinMaxIndex::store( +IMergeTreeDataPart::MinMaxIndex::WrittenFiles IMergeTreeDataPart::MinMaxIndex::store( const Names & column_names, const DataTypes & data_types, const DiskPtr & disk_, @@ -121,6 +122,8 @@ void IMergeTreeDataPart::MinMaxIndex::store( throw Exception("Attempt to store uninitialized MinMax index for part " + part_path + ". This is a bug.", ErrorCodes::LOGICAL_ERROR); + WrittenFiles written_files; + for (size_t i = 0; i < column_names.size(); ++i) { String file_name = "minmax_" + escapeForFileName(column_names[i]) + ".idx"; @@ -133,8 +136,11 @@ void IMergeTreeDataPart::MinMaxIndex::store( out_hashing.next(); out_checksums.files[file_name].file_size = out_hashing.count(); out_checksums.files[file_name].file_hash = out_hashing.getHash(); - out->finalize(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } + + return written_files; } void IMergeTreeDataPart::MinMaxIndex::update(const Block & block, const Names & column_names) @@ -1117,6 +1123,7 @@ UInt64 IMergeTreeDataPart::calculateTotalSizeOnDisk(const DiskPtr & disk_, const void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_new_dir_if_exists) const +try { assertOnDisk(); @@ -1153,6 +1160,18 @@ void IMergeTreeDataPart::renameTo(const String & new_relative_path, bool remove_ storage.lockSharedData(*this); } +catch (...) +{ + if (startsWith(new_relative_path, "detached/")) + { + // Don't throw when the destination is to the detached folder. It might be able to + // recover in some cases, such as fetching parts into multi-disks while some of the + // disks are broken. + tryLogCurrentException(__PRETTY_FUNCTION__); + } + else + throw; +} void IMergeTreeDataPart::cleanupOldName(const String & old_part_name) const { @@ -1168,16 +1187,7 @@ std::optional IMergeTreeDataPart::keepSharedDataInDecoupledStorage() const if (force_keep_shared_data) return true; - /// TODO Unlocking in try-catch and ignoring exception look ugly - try - { - return !storage.unlockSharedData(*this); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__, "There is a problem with deleting part " + name + " from filesystem"); - } - return {}; + return !storage.unlockSharedData(*this); } void IMergeTreeDataPart::remove() const @@ -1266,6 +1276,7 @@ void IMergeTreeDataPart::remove() const try { /// Remove each expected file in directory, then remove directory itself. + IDisk::RemoveBatchRequest request; #if !defined(__clang__) # pragma GCC diagnostic push @@ -1274,18 +1285,19 @@ void IMergeTreeDataPart::remove() const for (const auto & [file, _] : checksums.files) { if (projection_directories.find(file) == projection_directories.end()) - disk->removeSharedFile(fs::path(to) / file, *keep_shared_data); + request.emplace_back(fs::path(to) / file); } #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(fs::path(to) / file, *keep_shared_data); + request.emplace_back(fs::path(to) / file); - disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, *keep_shared_data); - disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, *keep_shared_data); + request.emplace_back(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, true); + request.emplace_back(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, true); + disk->removeSharedFiles(request, *keep_shared_data); disk->removeDirectory(to); } catch (...) @@ -1319,22 +1331,24 @@ void IMergeTreeDataPart::projectionRemove(const String & parent_to, bool keep_sh try { /// Remove each expected file in directory, then remove directory itself. + IDisk::RemoveBatchRequest request; #if !defined(__clang__) # pragma GCC diagnostic push # pragma GCC diagnostic ignored "-Wunused-variable" #endif for (const auto & [file, _] : checksums.files) - disk->removeSharedFile(fs::path(to) / file, keep_shared_data); + request.emplace_back(fs::path(to) / file); #if !defined(__clang__) # pragma GCC diagnostic pop #endif for (const auto & file : {"checksums.txt", "columns.txt"}) - disk->removeSharedFile(fs::path(to) / file, keep_shared_data); - disk->removeSharedFileIfExists(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, keep_shared_data); - disk->removeSharedFileIfExists(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, keep_shared_data); + request.emplace_back(fs::path(to) / file); + request.emplace_back(fs::path(to) / DEFAULT_COMPRESSION_CODEC_FILE_NAME, true); + request.emplace_back(fs::path(to) / DELETE_ON_DESTROY_MARKER_FILE_NAME, true); + disk->removeSharedFiles(request, keep_shared_data); disk->removeSharedRecursive(to, keep_shared_data); } catch (...) @@ -1414,7 +1428,7 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di if (disk->exists(fs::path(path_to_clone) / relative_path)) { - LOG_WARNING(storage.log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); + LOG_WARNING(storage.log, "Path {} already exists. Will remove it and clone again.", fullPath(disk, path_to_clone + relative_path)); disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); @@ -1619,24 +1633,24 @@ String IMergeTreeDataPart::getUniqueId() const if (!disk->supportZeroCopyReplication()) throw Exception(fmt::format("Disk {} doesn't support zero-copy replication", disk->getName()), ErrorCodes::LOGICAL_ERROR); - String id = disk->getUniqueId(fs::path(getFullRelativePath()) / "checksums.txt"); - return id; + return disk->getUniqueId(fs::path(getFullRelativePath()) / FILE_FOR_REFERENCES_CHECK); } - -UInt32 IMergeTreeDataPart::getNumberOfRefereneces() const -{ - return volume->getDisk()->getRefCount(fs::path(getFullRelativePath()) / "checksums.txt"); -} - - -String IMergeTreeDataPart::getZeroLevelPartBlockID() const +String IMergeTreeDataPart::getZeroLevelPartBlockID(std::string_view token) const { if (info.level != 0) throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to get block id for non zero level part {}", name); SipHash hash; - checksums.computeTotalChecksumDataOnly(hash); + if (token.empty()) + { + checksums.computeTotalChecksumDataOnly(hash); + } + else + { + hash.update(token.data(), token.size()); + } + union { char bytes[16]; diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index e3370ad731e..85088b10f70 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -177,7 +177,8 @@ public: bool isEmpty() const { return rows_count == 0; } /// Compute part block id for zero level part. Otherwise throws an exception. - String getZeroLevelPartBlockID() const; + /// If token is not empty, block id is calculated based on it instead of block data + String getZeroLevelPartBlockID(std::string_view token) const; const MergeTreeData & storage; @@ -299,9 +300,11 @@ public: { } + using WrittenFiles = std::vector>; + void load(const MergeTreeData & data, const DiskPtr & disk_, const String & part_path); - void store(const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; - void store(const Names & column_names, const DataTypes & data_types, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; + [[nodiscard]] WrittenFiles store(const MergeTreeData & data, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; + [[nodiscard]] WrittenFiles store(const Names & column_names, const DataTypes & data_types, const DiskPtr & disk_, const String & part_path, Checksums & checksums) const; void update(const Block & block, const Names & column_names); void merge(const MinMaxIndex & other); @@ -405,6 +408,18 @@ public: /// (number of rows, number of rows with default values, etc). static inline constexpr auto SERIALIZATION_FILE_NAME = "serialization.json"; + /// One of part files which is used to check how many references (I'd like + /// to say hardlinks, but it will confuse even more) we have for the part + /// for zero copy replication. Sadly it's very complex. + /// + /// NOTE: it's not a random "metadata" file for part like 'columns.txt'. If + /// two relative parts (for example all_1_1_0 and all_1_1_0_100) has equal + /// checksums.txt it means that one part was obtained by FREEZE operation or + /// it was mutation without any change for source part. In this case we + /// really don't need to remove data from remote FS and need only decrement + /// reference counter locally. + static inline constexpr auto FILE_FOR_REFERENCES_CHECK = "checksums.txt"; + /// Checks that all TTLs (table min/max, column ttls, so on) for part /// calculated. Part without calculated TTL may exist if TTL was added after /// part creation (using alter query with materialize_ttl setting). @@ -414,10 +429,6 @@ public: /// Required for distinguish different copies of the same part on remote FS. String getUniqueId() const; - /// Return hardlink count for part. - /// Required for keep data on remote FS when part has shadow copies. - UInt32 getNumberOfRefereneces() const; - protected: /// Total size of all columns, calculated once in calcuateColumnSizesOnDisk diff --git a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h index d0d3f283478..34c53eda846 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPartWriter.h +++ b/src/Storages/MergeTree/IMergeTreeDataPartWriter.h @@ -32,7 +32,9 @@ public: virtual void write(const Block & block, const IColumn::Permutation * permutation) = 0; - virtual void finish(IMergeTreeDataPart::Checksums & checksums, bool sync) = 0; + virtual void fillChecksums(IMergeTreeDataPart::Checksums & checksums) = 0; + + virtual void finish(bool sync) = 0; Columns releaseIndexColumns(); const MergeTreeIndexGranularity & getIndexGranularity() const { return index_granularity; } diff --git a/src/Storages/MergeTree/IMergeTreeReader.cpp b/src/Storages/MergeTree/IMergeTreeReader.cpp index 79186402027..87cda721862 100644 --- a/src/Storages/MergeTree/IMergeTreeReader.cpp +++ b/src/Storages/MergeTree/IMergeTreeReader.cpp @@ -192,6 +192,7 @@ void IMergeTreeReader::evaluateMissingDefaults(Block additional_columns, Columns additional_columns, columns, metadata_snapshot->getColumns(), storage.getContext()); if (dag) { + dag->addMaterializingOutputActions(); auto actions = std::make_shared< ExpressionActions>(std::move(dag), ExpressionActionsSettings::fromSettings(storage.getContext()->getSettingsRef())); diff --git a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp index 5393d71ff86..b4a902499db 100644 --- a/src/Storages/MergeTree/IMergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/IMergedBlockOutputStream.cpp @@ -79,10 +79,7 @@ NameSet IMergedBlockOutputStream::removeEmptyColumnsFromPart( for (const String & removed_file : remove_files) { if (checksums.files.count(removed_file)) - { - data_part->volume->getDisk()->removeFile(data_part->getFullRelativePath() + removed_file); checksums.files.erase(removed_file); - } } /// Remove columns from columns array diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 8ce65211e3e..323b59e2902 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -1292,8 +1292,8 @@ bool KeyCondition::tryParseAtomFromAST(const ASTPtr & node, ContextPtr context, key_expr_type_not_null = key_expr_type; bool cast_not_needed = is_set_const /// Set args are already casted inside Set::createFromAST - || ((isNativeNumber(key_expr_type_not_null) || isDateTime(key_expr_type_not_null)) - && (isNativeNumber(const_type) || isDateTime(const_type))); /// Numbers and DateTime are accurately compared without cast. + || ((isNativeInteger(key_expr_type_not_null) || isDateTime(key_expr_type_not_null)) + && (isNativeInteger(const_type) || isDateTime(const_type))); /// Native integers and DateTime are accurately compared without cast. if (!cast_not_needed && !key_expr_type_not_null->equals(*const_type)) { diff --git a/src/Storages/MergeTree/LeaderElection.h b/src/Storages/MergeTree/LeaderElection.h index aadaf953e73..b05026d52f9 100644 --- a/src/Storages/MergeTree/LeaderElection.h +++ b/src/Storages/MergeTree/LeaderElection.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -48,7 +49,7 @@ void checkNoOldLeaders(Poco::Logger * log, ZooKeeper & zookeeper, const String p } else { - std::sort(potential_leaders.begin(), potential_leaders.end()); + ::sort(potential_leaders.begin(), potential_leaders.end()); if (potential_leaders.front() == persistent_multiple_leaders) return; diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp index d92eaf85f3d..be3a872971e 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.cpp @@ -46,14 +46,10 @@ std::pair MergeFromLogEntryT /// In some use cases merging can be more expensive than fetching /// and it may be better to spread merges tasks across the replicas /// instead of doing exactly the same merge cluster-wise - std::optional replica_to_execute_merge; - bool replica_to_execute_merge_picked = false; if (storage.merge_strategy_picker.shouldMergeOnSingleReplica(entry)) { - replica_to_execute_merge = storage.merge_strategy_picker.pickReplicaToExecuteMerge(entry); - replica_to_execute_merge_picked = true; - + std::optional replica_to_execute_merge = storage.merge_strategy_picker.pickReplicaToExecuteMerge(entry); if (replica_to_execute_merge) { LOG_DEBUG(log, @@ -84,7 +80,7 @@ std::pair MergeFromLogEntryT /// 3. We have two intersecting parts, both cover source_part_name. It's logical error. /// TODO Why 1 and 2 can happen? Do we need more assertions here or somewhere else? constexpr const char * message = "Part {} is covered by {} but should be merged into {}. This shouldn't happen often."; - LOG_WARNING(log, message, source_part_name, source_part_or_covering->name, entry.new_part_name); + LOG_WARNING(log, fmt::runtime(message), source_part_name, source_part_or_covering->name, entry.new_part_name); if (!source_part_or_covering->info.contains(MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version))) throw Exception(ErrorCodes::LOGICAL_ERROR, message, source_part_name, source_part_or_covering->name, entry.new_part_name); return {false, {}}; @@ -158,22 +154,24 @@ std::pair MergeFromLogEntryT future_merged_part->updatePath(storage, reserved_space.get()); future_merged_part->merge_type = entry.merge_type; + if (storage_settings_ptr->allow_remote_fs_zero_copy_replication) { if (auto disk = reserved_space->getDisk(); disk->getType() == DB::DiskType::S3) { - if (storage.merge_strategy_picker.shouldMergeOnSingleReplicaShared(entry)) + String dummy; + if (!storage.findReplicaHavingCoveringPart(entry.new_part_name, true, dummy).empty()) { - if (!replica_to_execute_merge_picked) - replica_to_execute_merge = storage.merge_strategy_picker.pickReplicaToExecuteMerge(entry); + LOG_DEBUG(log, "Merge of part {} finished by some other replica, will fetch merged part", entry.new_part_name); + return {false, {}}; + } - if (replica_to_execute_merge) - { - LOG_DEBUG(log, - "Prefer fetching part {} from replica {} due s3_execute_merges_on_single_replica_time_threshold", - entry.new_part_name, replica_to_execute_merge.value()); - return {false, {}}; - } + zero_copy_lock = storage.tryCreateZeroCopyExclusiveLock(entry.new_part_name, disk); + + if (!zero_copy_lock) + { + LOG_DEBUG(log, "Merge of part {} started by some other replica, will wait it and fetch merged part", entry.new_part_name); + return {false, {}}; } } } @@ -271,6 +269,9 @@ bool MergeFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrite throw; } + if (zero_copy_lock) + zero_copy_lock->lock->unlock(); + /** Removing old parts from ZK and from the disk is delayed - see ReplicatedMergeTreeCleanupThread, clearOldParts. */ diff --git a/src/Storages/MergeTree/MergeFromLogEntryTask.h b/src/Storages/MergeTree/MergeFromLogEntryTask.h index da286b27b20..5fdee5defe9 100644 --- a/src/Storages/MergeTree/MergeFromLogEntryTask.h +++ b/src/Storages/MergeTree/MergeFromLogEntryTask.h @@ -8,6 +8,7 @@ #include #include #include +#include namespace DB @@ -37,6 +38,7 @@ private: MergeTreeData::DataPartsVector parts; MergeTreeData::TransactionUniquePtr transaction_ptr{nullptr}; + std::optional zero_copy_lock; StopwatchUniquePtr stopwatch_ptr{nullptr}; MergeTreeData::MutableDataPartPtr part; diff --git a/src/Storages/MergeTree/MergeTask.cpp b/src/Storages/MergeTree/MergeTask.cpp index 7d4b731551c..5d5035d62ca 100644 --- a/src/Storages/MergeTree/MergeTask.cpp +++ b/src/Storages/MergeTree/MergeTask.cpp @@ -478,9 +478,11 @@ void MergeTask::VerticalMergeStage::finalizeVerticalMergeForOneColumn() const throw Exception("Cancelled merging parts", ErrorCodes::ABORTED); ctx->executor.reset(); - auto changed_checksums = ctx->column_to->writeSuffixAndGetChecksums(global_ctx->new_data_part, global_ctx->checksums_gathered_columns, ctx->need_sync); + auto changed_checksums = ctx->column_to->fillChecksums(global_ctx->new_data_part, global_ctx->checksums_gathered_columns); global_ctx->checksums_gathered_columns.add(std::move(changed_checksums)); + ctx->delayed_streams.emplace_back(std::move(ctx->column_to)); + if (global_ctx->rows_written != ctx->column_elems_written) { throw Exception("Written " + toString(ctx->column_elems_written) + " elements of column " + column_name + @@ -505,9 +507,8 @@ void MergeTask::VerticalMergeStage::finalizeVerticalMergeForOneColumn() const bool MergeTask::VerticalMergeStage::finalizeVerticalMergeForAllColumns() const { - /// No need to execute this part if it is horizontal merge. - if (global_ctx->chosen_merge_algorithm != MergeAlgorithm::Vertical) - return false; + for (auto & stream : ctx->delayed_streams) + stream->finish(ctx->need_sync); return false; } @@ -633,9 +634,9 @@ bool MergeTask::MergeProjectionsStage::finalizeProjectionsAndWholeMerge() const } if (global_ctx->chosen_merge_algorithm != MergeAlgorithm::Vertical) - global_ctx->to->writeSuffixAndFinalizePart(global_ctx->new_data_part, ctx->need_sync); + global_ctx->to->finalizePart(global_ctx->new_data_part, ctx->need_sync); else - global_ctx->to->writeSuffixAndFinalizePart( + global_ctx->to->finalizePart( global_ctx->new_data_part, ctx->need_sync, &global_ctx->storage_columns, &global_ctx->checksums_gathered_columns); global_ctx->promise.set_value(global_ctx->new_data_part); diff --git a/src/Storages/MergeTree/MergeTask.h b/src/Storages/MergeTree/MergeTask.h index d8e48c1794a..aa64c4c2265 100644 --- a/src/Storages/MergeTree/MergeTask.h +++ b/src/Storages/MergeTree/MergeTask.h @@ -268,6 +268,7 @@ private: Float64 progress_before = 0; std::unique_ptr column_to{nullptr}; + std::vector> delayed_streams; size_t column_elems_written{0}; QueryPipeline column_parts_pipeline; std::unique_ptr executor; diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp index 797caff8f69..499a8fbbaa6 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp @@ -3,11 +3,19 @@ #include #include +#include #include + namespace DB { +namespace ErrorCodes +{ + extern const int ABORTED; +} + + template void MergeTreeBackgroundExecutor::wait() { @@ -86,12 +94,18 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) ALLOW_ALLOCATIONS_IN_SCOPE; need_execute_again = item->task->executeStep(); } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info. + LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false))); + else + tryLogCurrentException(__PRETTY_FUNCTION__); + } catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); } - if (need_execute_again) { std::lock_guard guard(mutex); @@ -118,7 +132,6 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) return; } - { std::lock_guard guard(mutex); erase_from_active(); @@ -132,12 +145,18 @@ void MergeTreeBackgroundExecutor::routine(TaskRuntimeDataPtr item) /// But it is rather safe, because we have try...catch block here, and another one in ThreadPool. item->task->onCompleted(); } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info. + LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false))); + else + tryLogCurrentException(__PRETTY_FUNCTION__); + } catch (...) { tryLogCurrentException(__PRETTY_FUNCTION__); } - /// We have to call reset() under a lock, otherwise a race is possible. /// Imagine, that task is finally completed (last execution returned false), /// we removed the task from both queues, but still have pointer. diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h index f4635812e08..5cfa7b6ed7c 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h @@ -159,7 +159,6 @@ template class MergeTreeBackgroundExecutor final : public shared_ptr_helper> { public: - MergeTreeBackgroundExecutor( String name_, size_t threads_count_, @@ -194,7 +193,6 @@ public: void wait(); private: - String name; size_t threads_count{0}; size_t max_tasks_count{0}; @@ -210,6 +208,7 @@ private: std::condition_variable has_tasks; std::atomic_bool shutdown{false}; ThreadPool pool; + Poco::Logger * log = &Poco::Logger::get("MergeTreeBackgroundExecutor"); }; extern template class MergeTreeBackgroundExecutor; diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 9a455c2d93c..23d44d177b7 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -67,7 +68,7 @@ #include #include -#include +#include #include #include @@ -264,9 +265,21 @@ MergeTreeData::MergeTreeData( /// Creating directories, if not exist. for (const auto & disk : getDisks()) { + /// TODO: implement it the main issue in DataPartsExchange (not able to send directories metadata) + if (supportsReplication() && settings->allow_remote_fs_zero_copy_replication + && disk->supportZeroCopyReplication() && metadata_.hasProjections()) + { + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Projections are not supported when zero-copy replication is enabled for table. " + "Currently disk '{}' supports zero copy replication", disk->getName()); + } + + if (disk->isBroken()) + continue; + disk->createDirectories(relative_data_path); disk->createDirectories(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME); String current_version_file_path = fs::path(relative_data_path) / MergeTreeData::FORMAT_VERSION_FILE_NAME; + if (disk->exists(current_version_file_path)) { if (!version_file.first.empty()) @@ -645,13 +658,14 @@ void MergeTreeData::checkTTLExpressions(const StorageInMemoryMetadata & new_meta { for (const auto & move_ttl : new_table_ttl.move_ttl) { - if (!getDestinationForMoveTTL(move_ttl)) + if (!move_ttl.if_exists && !getDestinationForMoveTTL(move_ttl)) { String message; if (move_ttl.destination_type == DataDestinationType::DISK) - message = "No such disk " + backQuote(move_ttl.destination_name) + " for given storage policy."; + message = "No such disk " + backQuote(move_ttl.destination_name) + " for given storage policy"; else - message = "No such volume " + backQuote(move_ttl.destination_name) + " for given storage policy."; + message = "No such volume " + backQuote(move_ttl.destination_name) + " for given storage policy"; + throw Exception(message, ErrorCodes::BAD_TTL_EXPRESSION); } } @@ -1195,6 +1209,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) for (const auto & [disk_name, disk] : getContext()->getDisksMap()) { + if (disk->isBroken()) + continue; + if (defined_disk_names.count(disk_name) == 0 && disk->exists(relative_data_path)) { for (const auto it = disk->iterateDirectory(relative_data_path); it->isValid(); it->next()) @@ -1215,6 +1232,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) std::mutex wal_init_lock; for (const auto & disk_ptr : disks) { + if (disk_ptr->isBroken()) + continue; + auto & disk_parts = disk_part_map[disk_ptr->getName()]; auto & disk_wal_parts = disk_wal_part_map[disk_ptr->getName()]; @@ -1276,7 +1296,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) return; } - DataPartsVector broken_parts_to_detach; DataPartsVector duplicate_parts_to_remove; @@ -1287,6 +1306,9 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) if (!parts_from_wal.empty()) loadDataPartsFromWAL(broken_parts_to_detach, duplicate_parts_to_remove, parts_from_wal, part_lock); + for (auto & part : duplicate_parts_to_remove) + part->remove(); + for (auto & part : broken_parts_to_detach) part->renameToDetached("broken-on-start"); /// detached parts must not have '_' in prefixes @@ -1344,7 +1366,6 @@ void MergeTreeData::loadDataParts(bool skip_sanity_checks) calculateColumnAndSecondaryIndexSizesImpl(); - LOG_DEBUG(log, "Loaded data parts ({} items)", data_parts_indexes.size()); } @@ -1381,6 +1402,9 @@ size_t MergeTreeData::clearOldTemporaryDirectories(const MergeTreeDataMergerMuta /// Delete temporary directories older than a day. for (const auto & disk : getDisks()) { + if (disk->isBroken()) + continue; + for (auto it = disk->iterateDirectory(relative_data_path); it->isValid(); it->next()) { const std::string & basename = it->name(); @@ -1590,12 +1614,8 @@ void MergeTreeData::clearPartsFromFilesystem(const DataPartsVector & parts_to_re { pool.scheduleOrThrowOnError([&, thread_group = CurrentThread::getGroup()] { - SCOPE_EXIT_SAFE( - if (thread_group) - CurrentThread::detachQueryIfNotDetached(); - ); if (thread_group) - CurrentThread::attachTo(thread_group); + CurrentThread::attachToIfDetached(thread_group); LOG_DEBUG(log, "Removing part from filesystem {}", part->name); part->remove(); @@ -1627,7 +1647,7 @@ size_t MergeTreeData::clearOldWriteAheadLogs() if (all_block_numbers_on_disk.empty()) return 0; - std::sort(all_block_numbers_on_disk.begin(), all_block_numbers_on_disk.end()); + ::sort(all_block_numbers_on_disk.begin(), all_block_numbers_on_disk.end()); block_numbers_on_disk.push_back(all_block_numbers_on_disk[0]); for (size_t i = 1; i < all_block_numbers_on_disk.size(); ++i) { @@ -1658,12 +1678,14 @@ size_t MergeTreeData::clearOldWriteAheadLogs() for (auto disk_it = disks.rbegin(); disk_it != disks.rend(); ++disk_it) { auto disk_ptr = *disk_it; + if (disk_ptr->isBroken()) + continue; for (auto it = disk_ptr->iterateDirectory(relative_data_path); it->isValid(); it->next()) { auto min_max_block_number = MergeTreeWriteAheadLog::tryParseMinMaxBlockNumber(it->name()); if (min_max_block_number && is_range_on_disk(min_max_block_number->first, min_max_block_number->second)) { - LOG_DEBUG(log, "Removing from filesystem the outdated WAL file " + it->name()); + LOG_DEBUG(log, "Removing from filesystem the outdated WAL file {}", it->name()); disk_ptr->removeFile(relative_data_path + it->name()); ++cleared_count; } @@ -1740,6 +1762,9 @@ void MergeTreeData::dropAllData() for (const auto & disk : getDisks()) { + if (disk->isBroken()) + continue; + try { disk->removeRecursive(relative_data_path); @@ -1774,6 +1799,8 @@ void MergeTreeData::dropIfEmpty() { for (const auto & disk : getDisks()) { + if (disk->isBroken()) + continue; /// Non recursive, exception is thrown if there are more files. disk->removeFileIfExists(fs::path(relative_data_path) / MergeTreeData::FORMAT_VERSION_FILE_NAME); disk->removeDirectory(fs::path(relative_data_path) / MergeTreeData::DETACHED_DIR_NAME); @@ -2014,11 +2041,26 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context "ALTER ADD INDEX is not supported for tables with the old syntax", ErrorCodes::BAD_ARGUMENTS); } - if (command.type == AlterCommand::ADD_PROJECTION && !is_custom_partitioned) + if (command.type == AlterCommand::ADD_PROJECTION) + { - throw Exception( - "ALTER ADD PROJECTION is not supported for tables with the old syntax", - ErrorCodes::BAD_ARGUMENTS); + if (!is_custom_partitioned) + throw Exception( + "ALTER ADD PROJECTION is not supported for tables with the old syntax", + ErrorCodes::BAD_ARGUMENTS); + + /// TODO: implement it the main issue in DataPartsExchange (not able to send directories metadata) + if (supportsReplication() && getSettings()->allow_remote_fs_zero_copy_replication) + { + auto storage_policy = getStoragePolicy(); + auto disks = storage_policy->getDisks(); + for (const auto & disk : disks) + { + if (disk->supportZeroCopyReplication()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ALTER ADD PROJECTION is not supported when zero-copy replication is enabled for table. " + "Currently disk '{}' supports zero copy replication", disk->getName()); + } + } } if (command.type == AlterCommand::RENAME_COLUMN) { @@ -2442,7 +2484,12 @@ MergeTreeData::DataPartsVector MergeTreeData::getActivePartsToReplace( } -bool MergeTreeData::renameTempPartAndAdd(MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction, MergeTreeDeduplicationLog * deduplication_log) +bool MergeTreeData::renameTempPartAndAdd( + MutableDataPartPtr & part, + SimpleIncrement * increment, + Transaction * out_transaction, + MergeTreeDeduplicationLog * deduplication_log, + std::string_view deduplication_token) { if (out_transaction && &out_transaction->data != this) throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", @@ -2451,7 +2498,7 @@ bool MergeTreeData::renameTempPartAndAdd(MutableDataPartPtr & part, SimpleIncrem DataPartsVector covered_parts; { auto lock = lockParts(); - if (!renameTempPartAndReplace(part, increment, out_transaction, lock, &covered_parts, deduplication_log)) + if (!renameTempPartAndReplace(part, increment, out_transaction, lock, &covered_parts, deduplication_log, deduplication_token)) return false; } if (!covered_parts.empty()) @@ -2463,8 +2510,13 @@ bool MergeTreeData::renameTempPartAndAdd(MutableDataPartPtr & part, SimpleIncrem bool MergeTreeData::renameTempPartAndReplace( - MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction, - std::unique_lock & lock, DataPartsVector * out_covered_parts, MergeTreeDeduplicationLog * deduplication_log) + MutableDataPartPtr & part, + SimpleIncrement * increment, + Transaction * out_transaction, + std::unique_lock & lock, + DataPartsVector * out_covered_parts, + MergeTreeDeduplicationLog * deduplication_log, + std::string_view deduplication_token) { if (out_transaction && &out_transaction->data != this) throw Exception("MergeTreeData::Transaction for one table cannot be used with another. It is a bug.", @@ -2526,7 +2578,7 @@ bool MergeTreeData::renameTempPartAndReplace( /// deduplication. if (deduplication_log) { - String block_id = part->getZeroLevelPartBlockID(); + const String block_id = part->getZeroLevelPartBlockID(deduplication_token); auto res = deduplication_log->addPart(block_id, part_info); if (!res.second) { @@ -3335,9 +3387,6 @@ void MergeTreeData::movePartitionToDisk(const ASTPtr & partition, const String & parts = getDataPartsVectorInPartition(MergeTreeDataPartState::Active, partition_id); auto disk = getStoragePolicy()->getDiskByName(name); - if (!disk) - throw Exception("Disk " + name + " does not exists on policy " + getStoragePolicy()->getName(), ErrorCodes::UNKNOWN_DISK); - parts.erase(std::remove_if(parts.begin(), parts.end(), [&](auto part_ptr) { return part_ptr->volume->getDisk()->getName() == disk->getName(); @@ -4084,10 +4133,10 @@ ReservationPtr MergeTreeData::tryReserveSpacePreferringTTLRules( SpacePtr destination_ptr = getDestinationForMoveTTL(*move_ttl_entry, is_insert); if (!destination_ptr) { - if (move_ttl_entry->destination_type == DataDestinationType::VOLUME) + if (move_ttl_entry->destination_type == DataDestinationType::VOLUME && !move_ttl_entry->if_exists) LOG_WARNING(log, "Would like to reserve space on volume '{}' by TTL rule of table '{}' but volume was not found or rule is not applicable at the moment", move_ttl_entry->destination_name, log_name); - else if (move_ttl_entry->destination_type == DataDestinationType::DISK) + else if (move_ttl_entry->destination_type == DataDestinationType::DISK && !move_ttl_entry->if_exists) LOG_WARNING(log, "Would like to reserve space on disk '{}' by TTL rule of table '{}' but disk was not found or rule is not applicable at the moment", move_ttl_entry->destination_name, log_name); } @@ -4121,7 +4170,7 @@ SpacePtr MergeTreeData::getDestinationForMoveTTL(const TTLDescription & move_ttl auto policy = getStoragePolicy(); if (move_ttl.destination_type == DataDestinationType::VOLUME) { - auto volume = policy->getVolumeByName(move_ttl.destination_name); + auto volume = policy->tryGetVolumeByName(move_ttl.destination_name); if (!volume) return {}; @@ -4133,7 +4182,8 @@ SpacePtr MergeTreeData::getDestinationForMoveTTL(const TTLDescription & move_ttl } else if (move_ttl.destination_type == DataDestinationType::DISK) { - auto disk = policy->getDiskByName(move_ttl.destination_name); + auto disk = policy->tryGetDiskByName(move_ttl.destination_name); + if (!disk) return {}; @@ -4477,6 +4527,7 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( const SelectQueryInfo & query_info, const DataPartsVector & parts, DataPartsVector & normal_parts, + const PartitionIdToMaxBlock * max_block_numbers_to_read, ContextPtr query_context) const { if (!metadata_snapshot->minmax_count_projection) @@ -4513,6 +4564,23 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( if (virtual_columns_block.rows() == 0) return {}; + std::optional partition_pruner; + std::optional minmax_idx_condition; + DataTypes minmax_columns_types; + if (metadata_snapshot->hasPartitionKey()) + { + const auto & partition_key = metadata_snapshot->getPartitionKey(); + auto minmax_columns_names = getMinMaxColumnsNames(partition_key); + minmax_columns_types = getMinMaxColumnsTypes(partition_key); + + minmax_idx_condition.emplace( + query_info, + query_context, + minmax_columns_names, + getMinMaxExpr(partition_key, ExpressionActionsSettings::fromContext(query_context))); + partition_pruner.emplace(metadata_snapshot, query_info, query_context, false /* strict */); + } + // Generate valid expressions for filtering VirtualColumnUtils::prepareFilterBlockWithQuery(query_info.query, query_context, virtual_columns_block, expression_ast); if (expression_ast) @@ -4521,6 +4589,8 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( size_t rows = virtual_columns_block.rows(); const ColumnString & part_name_column = typeid_cast(*virtual_columns_block.getByName("_part").column); size_t part_idx = 0; + auto filter_column = ColumnUInt8::create(); + auto & filter_column_data = filter_column->getData(); for (size_t row = 0; row < rows; ++row) { while (parts[part_idx]->name != part_name_column.getDataAt(row)) @@ -4531,12 +4601,32 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( if (!part->minmax_idx->initialized) throw Exception("Found a non-empty part with uninitialized minmax_idx. It's a bug", ErrorCodes::LOGICAL_ERROR); + filter_column_data.emplace_back(); + + if (max_block_numbers_to_read) + { + auto blocks_iterator = max_block_numbers_to_read->find(part->info.partition_id); + if (blocks_iterator == max_block_numbers_to_read->end() || part->info.max_block > blocks_iterator->second) + continue; + } + + if (minmax_idx_condition + && !minmax_idx_condition->checkInHyperrectangle(part->minmax_idx->hyperrectangle, minmax_columns_types).can_be_true) + continue; + + if (partition_pruner) + { + if (partition_pruner->canBePruned(*part)) + continue; + } + if (need_primary_key_max_column && !part->index_granularity.hasFinalMark()) { normal_parts.push_back(part); continue; } + filter_column_data.back() = 1; size_t pos = 0; for (size_t i : metadata_snapshot->minmax_count_projection->partition_value_indices) { @@ -4579,6 +4669,16 @@ Block MergeTreeData::getMinMaxCountProjectionBlock( } block.setColumns(std::move(partition_minmax_count_columns)); + FilterDescription filter(*filter_column); + for (size_t i = 0; i < virtual_columns_block.columns(); ++i) + { + ColumnPtr & column = virtual_columns_block.safeGetByPosition(i).column; + column = column->filter(*filter.data, -1); + } + + if (block.rows() == 0) + return {}; + Block res; for (const auto & name : required_columns) { @@ -4603,21 +4703,31 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (!settings.allow_experimental_projection_optimization || query_info.ignore_projections || query_info.is_projection_query) return std::nullopt; - const auto & query_ptr = query_info.original_query; + // Currently projections don't support parallel replicas reading yet. + if (settings.parallel_replicas_count > 1 || settings.max_parallel_replicas > 1) + return std::nullopt; - if (auto * select = query_ptr->as(); select) - { - // Currently projections don't support final yet. - if (select->final()) - return std::nullopt; + auto query_ptr = query_info.original_query; + auto * select_query = query_ptr->as(); + if (!select_query) + return std::nullopt; - // Currently projections don't support ARRAY JOIN yet. - if (select->arrayJoinExpressionList().first) - return std::nullopt; - } + // Currently projections don't support final yet. + if (select_query->final()) + return std::nullopt; - // Currently projections don't support sampling yet. - if (settings.parallel_replicas_count > 1) + // Currently projections don't support sample yet. + if (select_query->sampleSize()) + return std::nullopt; + + // Currently projections don't support ARRAY JOIN yet. + if (select_query->arrayJoinExpressionList().first) + return std::nullopt; + + // In order to properly analyze joins, aliases should be recognized. However, aliases get lost during projection analysis. + // Let's disable projection if there are any JOIN clauses. + // TODO: We need a better identifier resolution mechanism for projection analysis. + if (select_query->join()) return std::nullopt; InterpreterSelectQuery select( @@ -4652,7 +4762,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg keys.insert(desc.name); key_name_pos_map.insert({desc.name, pos++}); } - auto actions_settings = ExpressionActionsSettings::fromSettings(settings); + auto actions_settings = ExpressionActionsSettings::fromSettings(settings, CompileExpressions::yes); // All required columns should be provided by either current projection or previous actions // Let's traverse backward to finish the check. @@ -4688,7 +4798,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg required_columns.erase(column.name); { - // Prewhere_action should not add missing keys. + // prewhere_action should not add missing keys. auto new_prewhere_required_columns = prewhere_actions->foldActionsByProjection( prewhere_required_columns, projection.sample_block_for_keys, candidate.prewhere_info->prewhere_column_name, false); if (new_prewhere_required_columns.empty() && !prewhere_required_columns.empty()) @@ -4700,6 +4810,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (candidate.prewhere_info->row_level_filter) { auto row_level_filter_actions = candidate.prewhere_info->row_level_filter->clone(); + // row_level_filter_action should not add missing keys. auto new_prewhere_required_columns = row_level_filter_actions->foldActionsByProjection( prewhere_required_columns, projection.sample_block_for_keys, candidate.prewhere_info->row_level_column_name, false); if (new_prewhere_required_columns.empty() && !prewhere_required_columns.empty()) @@ -4711,6 +4822,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg if (candidate.prewhere_info->alias_actions) { auto alias_actions = candidate.prewhere_info->alias_actions->clone(); + // alias_action should not add missing keys. auto new_prewhere_required_columns = alias_actions->foldActionsByProjection(prewhere_required_columns, projection.sample_block_for_keys, {}, false); if (new_prewhere_required_columns.empty() && !prewhere_required_columns.empty()) @@ -4748,6 +4860,18 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg sample_block_for_keys.insertUnique(column); } + // If optimize_aggregation_in_order = true, we need additional information to transform the projection's pipeline. + auto attach_aggregation_in_order_info = [&]() + { + for (const auto & key : keys) + { + auto actions_dag = analysis_result.before_aggregation->clone(); + actions_dag->foldActionsByProjection({key}, sample_block_for_keys); + candidate.group_by_elements_actions.emplace_back(std::make_shared(actions_dag, actions_settings)); + candidate.group_by_elements_order_descr.emplace_back(key, 1, 1); + } + }; + if (projection.type == ProjectionDescription::Type::Aggregate && analysis_result.need_aggregate && can_use_aggregate_projection) { bool match = true; @@ -4757,16 +4881,13 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg { const auto * column = sample_block.findByName(aggregate.column_name); if (column) - { aggregates.insert(*column); - } else { match = false; break; } } - if (!match) return; @@ -4782,14 +4903,7 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg return; if (analysis_result.optimize_aggregation_in_order) - { - for (const auto & key : keys) - { - auto actions_dag = analysis_result.before_aggregation->clone(); - actions_dag->foldActionsByProjection({key}, sample_block_for_keys); - candidate.group_by_elements_actions.emplace_back(std::make_shared(actions_dag, actions_settings)); - } - } + attach_aggregation_in_order_info(); // Reorder aggregation keys and attach aggregates candidate.before_aggregation->reorderAggregationKeysForProjection(key_name_pos_map); @@ -4803,19 +4917,24 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg candidates.push_back(std::move(candidate)); } } - - if (projection.type == ProjectionDescription::Type::Normal && (analysis_result.hasWhere() || analysis_result.hasPrewhere())) + else if (projection.type == ProjectionDescription::Type::Normal) { - const auto & actions - = analysis_result.before_aggregation ? analysis_result.before_aggregation : analysis_result.before_order_by; - NameSet required_columns; - for (const auto & column : actions->getRequiredColumns()) - required_columns.insert(column.name); + if (analysis_result.before_aggregation && analysis_result.optimize_aggregation_in_order) + attach_aggregation_in_order_info(); - if (rewrite_before_where(candidate, projection, required_columns, sample_block, {})) + if (analysis_result.hasWhere() || analysis_result.hasPrewhere()) { - candidate.required_columns = {required_columns.begin(), required_columns.end()}; - candidates.push_back(std::move(candidate)); + const auto & actions + = analysis_result.before_aggregation ? analysis_result.before_aggregation : analysis_result.before_order_by; + NameSet required_columns; + for (const auto & column : actions->getRequiredColumns()) + required_columns.insert(column.name); + + if (rewrite_before_where(candidate, projection, required_columns, sample_block, {})) + { + candidate.required_columns = {required_columns.begin(), required_columns.end()}; + candidates.push_back(std::move(candidate)); + } } } }; @@ -4844,9 +4963,15 @@ std::optional MergeTreeData::getQueryProcessingStageWithAgg { DataPartsVector normal_parts; query_info.minmax_count_projection_block = getMinMaxCountProjectionBlock( - metadata_snapshot, minmax_conut_projection_candidate->required_columns, query_info, parts, normal_parts, query_context); + metadata_snapshot, + minmax_conut_projection_candidate->required_columns, + query_info, + parts, + normal_parts, + max_added_blocks.get(), + query_context); - if (minmax_conut_projection_candidate->prewhere_info) + if (query_info.minmax_count_projection_block && minmax_conut_projection_candidate->prewhere_info) { const auto & prewhere_info = minmax_conut_projection_candidate->prewhere_info; if (prewhere_info->alias_actions) @@ -5144,6 +5269,23 @@ Strings MergeTreeData::getDataPaths() const } +void MergeTreeData::reportBrokenPart(MergeTreeData::DataPartPtr & data_part) const +{ + if (data_part->volume && data_part->volume->getDisk()->isBroken()) + { + auto disk = data_part->volume->getDisk(); + auto parts = getDataParts(); + LOG_WARNING(log, "Scanning parts to recover on broken disk {}.", disk->getName() + "@" + disk->getPath()); + for (const auto & part : parts) + { + if (part->volume && part->volume->getDisk()->getName() == disk->getName()) + broken_part_callback(part->name); + } + } + else + broken_part_callback(data_part->name); +} + MergeTreeData::MatcherFn MergeTreeData::getPartitionMatcher(const ASTPtr & partition_ast, ContextPtr local_context) const { bool prefixed = false; @@ -5577,7 +5719,7 @@ bool MergeTreeData::moveParts(const CurrentlyMovingPartsTaggerPtr & moving_tagge /// replica will actually move the part from disk to some /// zero-copy storage other replicas will just fetch /// metainformation. - if (auto lock = tryCreateZeroCopyExclusiveLock(moving_part.part, disk); lock) + if (auto lock = tryCreateZeroCopyExclusiveLock(moving_part.part->name, disk); lock) { cloned_part = parts_mover.clonePart(moving_part); parts_mover.swapClonedPart(cloned_part); @@ -5872,7 +6014,7 @@ ReservationPtr MergeTreeData::balancedReservation( writeCString("\nbalancer: \n", log_str); for (const auto & [disk_name, per_disk_parts] : disk_parts_for_logging) writeString(fmt::format(" {}: [{}]\n", disk_name, fmt::join(per_disk_parts, ", ")), log_str); - LOG_DEBUG(log, log_str.str()); + LOG_DEBUG(log, fmt::runtime(log_str.str())); if (ttl_infos) reserved_space = tryReserveSpacePreferringTTLRules( diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index ebd1950a720..1a04b2a389b 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -43,6 +43,7 @@ class MergeTreePartsMover; class MergeTreeDataMergerMutator; class MutationCommands; class Context; +using PartitionIdToMaxBlock = std::unordered_map; struct JobAndPool; struct ZeroCopyLock; @@ -391,6 +392,7 @@ public: const SelectQueryInfo & query_info, const DataPartsVector & parts, DataPartsVector & normal_parts, + const PartitionIdToMaxBlock * max_block_numbers_to_read, ContextPtr query_context) const; std::optional getQueryProcessingStageWithAggregateProjection( @@ -492,7 +494,12 @@ public: /// active set later with out_transaction->commit()). /// Else, commits the part immediately. /// Returns true if part was added. Returns false if part is covered by bigger part. - bool renameTempPartAndAdd(MutableDataPartPtr & part, SimpleIncrement * increment = nullptr, Transaction * out_transaction = nullptr, MergeTreeDeduplicationLog * deduplication_log = nullptr); + bool renameTempPartAndAdd( + MutableDataPartPtr & part, + SimpleIncrement * increment = nullptr, + Transaction * out_transaction = nullptr, + MergeTreeDeduplicationLog * deduplication_log = nullptr, + std::string_view deduplication_token = std::string_view()); /// The same as renameTempPartAndAdd but the block range of the part can contain existing parts. /// Returns all parts covered by the added part (in ascending order). @@ -502,9 +509,13 @@ public: /// Low-level version of previous one, doesn't lock mutex bool renameTempPartAndReplace( - MutableDataPartPtr & part, SimpleIncrement * increment, Transaction * out_transaction, DataPartsLock & lock, - DataPartsVector * out_covered_parts = nullptr, MergeTreeDeduplicationLog * deduplication_log = nullptr); - + MutableDataPartPtr & part, + SimpleIncrement * increment, + Transaction * out_transaction, + DataPartsLock & lock, + DataPartsVector * out_covered_parts = nullptr, + MergeTreeDeduplicationLog * deduplication_log = nullptr, + std::string_view deduplication_token = std::string_view()); /// Remove parts from working set immediately (without wait for background /// process). Transfer part state to temporary. Have very limited usage only @@ -596,6 +607,10 @@ public: broken_part_callback(name); } + /// Same as above but has the ability to check all other parts + /// which reside on the same disk of the suspicious part. + void reportBrokenPart(MergeTreeData::DataPartPtr & data_part) const; + /// TODO (alesap) Duplicate method required for compatibility. /// Must be removed. static ASTPtr extractKeyExpressionList(const ASTPtr & node) @@ -861,7 +876,7 @@ public: /// Lock part in zookeeper for shared data in several nodes /// Overridden in StorageReplicatedMergeTree - virtual void lockSharedData(const IMergeTreeDataPart &) const {} + virtual void lockSharedData(const IMergeTreeDataPart &, bool = false) const {} /// Unlock shared data part in zookeeper /// Overridden in StorageReplicatedMergeTree @@ -1184,7 +1199,7 @@ private: /// Create zero-copy exclusive lock for part and disk. Useful for coordination of /// distributed operations which can lead to data duplication. Implemented only in ReplicatedMergeTree. - virtual std::optional tryCreateZeroCopyExclusiveLock(const DataPartPtr &, const DiskPtr &) { return std::nullopt; } + virtual std::optional tryCreateZeroCopyExclusiveLock(const String &, const DiskPtr &) { return std::nullopt; } }; /// RAII struct to record big parts that are submerging or emerging. diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp index cb9fa7e6086..2f097b69fc4 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.cpp @@ -303,7 +303,6 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectPartsToMerge( SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinPartition( FutureMergedMutatedPartPtr future_part, - UInt64 & available_disk_space, const AllowedMergingPredicate & can_merge, const String & partition_id, bool final, @@ -355,6 +354,7 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti ++it; } + auto available_disk_space = data.getStoragePolicy()->getMaxUnreservedFreeSpace(); /// Enough disk space to cover the new merge with a margin. auto required_disk_space = sum_bytes * DISK_USAGE_COEFFICIENT_TO_SELECT; if (available_disk_space <= required_disk_space) @@ -382,7 +382,6 @@ SelectPartsDecision MergeTreeDataMergerMutator::selectAllPartsToMergeWithinParti LOG_DEBUG(log, "Selected {} parts from {} to {}", parts.size(), parts.front()->name, parts.back()->name); future_part->assign(std::move(parts)); - available_disk_space -= required_disk_space; return SelectPartsDecision::SELECTED; } diff --git a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h index bcac642eb16..82cad873dce 100644 --- a/src/Storages/MergeTree/MergeTreeDataMergerMutator.h +++ b/src/Storages/MergeTree/MergeTreeDataMergerMutator.h @@ -81,7 +81,6 @@ public: */ SelectPartsDecision selectAllPartsToMergeWithinPartition( FutureMergedMutatedPartPtr future_part, - UInt64 & available_disk_space, const AllowedMergingPredicate & can_merge, const String & partition_id, bool final, diff --git a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp index 4ec53d88339..e4a174a7d29 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartInMemory.cpp @@ -125,12 +125,12 @@ void MergeTreeDataPartInMemory::flushToDisk(const String & base_path, const Stri projection_compression_codec); projection_out.write(projection_part->block); - projection_out.writeSuffixAndFinalizePart(projection_data_part); + projection_out.finalizePart(projection_data_part, false); new_data_part->addProjectionPart(projection_name, std::move(projection_data_part)); } } - out.writeSuffixAndFinalizePart(new_data_part); + out.finalizePart(new_data_part, false); } void MergeTreeDataPartInMemory::makeCloneInDetached(const String & prefix, const StorageMetadataPtr & metadata_snapshot) const diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp index ce85bc75c80..d7b8f2c4165 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.cpp @@ -218,7 +218,7 @@ void MergeTreeDataPartWriterCompact::writeDataBlock(const Block & block, const G } } -void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterCompact::fillDataChecksums(IMergeTreeDataPart::Checksums & checksums) { if (columns_buffer.size() != 0) { @@ -253,6 +253,12 @@ void MergeTreeDataPartWriterCompact::finishDataSerialization(IMergeTreeDataPart: marks.next(); addToChecksums(checksums); + plain_file->preFinalize(); + marks_file->preFinalize(); +} + +void MergeTreeDataPartWriterCompact::finishDataSerialization(bool sync) +{ plain_file->finalize(); marks_file->finalize(); if (sync) @@ -356,16 +362,28 @@ size_t MergeTreeDataPartWriterCompact::ColumnsBuffer::size() const return accumulated_columns.at(0)->size(); } -void MergeTreeDataPartWriterCompact::finish(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterCompact::fillChecksums(IMergeTreeDataPart::Checksums & checksums) { // If we don't have anything to write, skip finalization. if (!columns_list.empty()) - finishDataSerialization(checksums, sync); + fillDataChecksums(checksums); if (settings.rewrite_primary_key) - finishPrimaryIndexSerialization(checksums, sync); + fillPrimaryIndexChecksums(checksums); - finishSkipIndicesSerialization(checksums, sync); + fillSkipIndicesChecksums(checksums); +} + +void MergeTreeDataPartWriterCompact::finish(bool sync) +{ + // If we don't have anything to write, skip finalization. + if (!columns_list.empty()) + finishDataSerialization(sync); + + if (settings.rewrite_primary_key) + finishPrimaryIndexSerialization(sync); + + finishSkipIndicesSerialization(sync); } } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h index 8b86a9701c9..cc33d8404c2 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterCompact.h @@ -20,11 +20,13 @@ public: void write(const Block & block, const IColumn::Permutation * permutation) override; - void finish(IMergeTreeDataPart::Checksums & checksums, bool sync) override; + void fillChecksums(IMergeTreeDataPart::Checksums & checksums) override; + void finish(bool sync) override; private: /// Finish serialization of the data. Flush rows in buffer to disk, compute checksums. - void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync); + void fillDataChecksums(IMergeTreeDataPart::Checksums & checksums); + void finishDataSerialization(bool sync); void fillIndexGranularity(size_t index_granularity_for_block, size_t rows_in_block) override; diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp index 5efcc63b2fc..0c715a7c27f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.cpp @@ -76,7 +76,7 @@ void MergeTreeDataPartWriterInMemory::calculateAndSerializePrimaryIndex(const Bl } } -void MergeTreeDataPartWriterInMemory::finish(IMergeTreeDataPart::Checksums & checksums, bool /* sync */) +void MergeTreeDataPartWriterInMemory::fillChecksums(IMergeTreeDataPart::Checksums & checksums) { /// If part is empty we still need to initialize block by empty columns. if (!part_in_memory->block) diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.h b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.h index 495585ad4d2..233ca81a697 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterInMemory.h @@ -18,7 +18,8 @@ public: /// You can write only one block. In-memory part can be written only at INSERT. void write(const Block & block, const IColumn::Permutation * permutation) override; - void finish(IMergeTreeDataPart::Checksums & checksums, bool sync) override; + void fillChecksums(IMergeTreeDataPart::Checksums & checksums) override; + void finish(bool /*sync*/) override {} private: void calculateAndSerializePrimaryIndex(const Block & primary_index_block); diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp index 2e299bb2447..8dca93f574f 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.cpp @@ -2,6 +2,7 @@ #include #include +#include "IO/WriteBufferFromFileDecorator.h" namespace DB { @@ -10,13 +11,24 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } -void MergeTreeDataPartWriterOnDisk::Stream::finalize() +void MergeTreeDataPartWriterOnDisk::Stream::preFinalize() { compressed.next(); /// 'compressed_buf' doesn't call next() on underlying buffer ('plain_hashing'). We should do it manually. plain_hashing.next(); marks.next(); + plain_file->preFinalize(); + marks_file->preFinalize(); + + is_prefinalized = true; +} + +void MergeTreeDataPartWriterOnDisk::Stream::finalize() +{ + if (!is_prefinalized) + preFinalize(); + plain_file->finalize(); marks_file->finalize(); } @@ -245,8 +257,7 @@ void MergeTreeDataPartWriterOnDisk::calculateAndSerializeSkipIndices(const Block } } -void MergeTreeDataPartWriterOnDisk::finishPrimaryIndexSerialization( - MergeTreeData::DataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterOnDisk::fillPrimaryIndexChecksums(MergeTreeData::DataPart::Checksums & checksums) { bool write_final_mark = (with_final_mark && data_written); if (write_final_mark && compute_granularity) @@ -269,6 +280,14 @@ void MergeTreeDataPartWriterOnDisk::finishPrimaryIndexSerialization( index_stream->next(); checksums.files["primary.idx"].file_size = index_stream->count(); checksums.files["primary.idx"].file_hash = index_stream->getHash(); + index_file_stream->preFinalize(); + } +} + +void MergeTreeDataPartWriterOnDisk::finishPrimaryIndexSerialization(bool sync) +{ + if (index_stream) + { index_file_stream->finalize(); if (sync) index_file_stream->sync(); @@ -276,8 +295,7 @@ void MergeTreeDataPartWriterOnDisk::finishPrimaryIndexSerialization( } } -void MergeTreeDataPartWriterOnDisk::finishSkipIndicesSerialization( - MergeTreeData::DataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterOnDisk::fillSkipIndicesChecksums(MergeTreeData::DataPart::Checksums & checksums) { for (size_t i = 0; i < skip_indices.size(); ++i) { @@ -288,8 +306,16 @@ void MergeTreeDataPartWriterOnDisk::finishSkipIndicesSerialization( for (auto & stream : skip_indices_streams) { - stream->finalize(); + stream->preFinalize(); stream->addToChecksums(checksums); + } +} + +void MergeTreeDataPartWriterOnDisk::finishSkipIndicesSerialization(bool sync) +{ + for (auto & stream : skip_indices_streams) + { + stream->finalize(); if (sync) stream->sync(); } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h index fb46175c2aa..5af8cbc1650 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterOnDisk.h @@ -71,6 +71,10 @@ public: std::unique_ptr marks_file; HashingWriteBuffer marks; + bool is_prefinalized = false; + + void preFinalize(); + void finalize(); void sync() const; @@ -107,9 +111,11 @@ protected: void calculateAndSerializeSkipIndices(const Block & skip_indexes_block, const Granules & granules_to_write); /// Finishes primary index serialization: write final primary index row (if required) and compute checksums - void finishPrimaryIndexSerialization(MergeTreeData::DataPart::Checksums & checksums, bool sync); + void fillPrimaryIndexChecksums(MergeTreeData::DataPart::Checksums & checksums); + void finishPrimaryIndexSerialization(bool sync); /// Finishes skip indices serialization: write all accumulated data to disk and compute checksums - void finishSkipIndicesSerialization(MergeTreeData::DataPart::Checksums & checksums, bool sync); + void fillSkipIndicesChecksums(MergeTreeData::DataPart::Checksums & checksums); + void finishSkipIndicesSerialization(bool sync); /// Get global number of the current which we are writing (or going to start to write) size_t getCurrentMark() const { return current_mark; } diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp index b620bf8130e..a3eec3e54bc 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.cpp @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB { @@ -411,6 +412,11 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai String escaped_name = escapeForFileName(name); String mrk_path = part_path + escaped_name + marks_file_extension; String bin_path = part_path + escaped_name + DATA_FILE_EXTENSION; + + /// Some columns may be removed because of ttl. Skip them. + if (!disk->exists(mrk_path)) + return; + auto mrk_in = disk->readFile(mrk_path); DB::CompressedReadBufferFromFile bin_in(disk->readFile(bin_path)); bool must_be_last = false; @@ -514,7 +520,7 @@ void MergeTreeDataPartWriterWide::validateColumnOfFixedSize(const NameAndTypePai } -void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterWide::fillDataChecksums(IMergeTreeDataPart::Checksums & checksums) { const auto & global_settings = storage.getContext()->getSettingsRef(); ISerialization::SerializeBinaryBulkSettings serialize_settings; @@ -547,10 +553,19 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch writeFinalMark(*it, offset_columns, serialize_settings.path); } } + + for (auto & stream : column_streams) + { + stream.second->preFinalize(); + stream.second->addToChecksums(checksums); + } +} + +void MergeTreeDataPartWriterWide::finishDataSerialization(bool sync) +{ for (auto & stream : column_streams) { stream.second->finalize(); - stream.second->addToChecksums(checksums); if (sync) stream.second->sync(); } @@ -574,16 +589,28 @@ void MergeTreeDataPartWriterWide::finishDataSerialization(IMergeTreeDataPart::Ch } -void MergeTreeDataPartWriterWide::finish(IMergeTreeDataPart::Checksums & checksums, bool sync) +void MergeTreeDataPartWriterWide::fillChecksums(IMergeTreeDataPart::Checksums & checksums) { // If we don't have anything to write, skip finalization. if (!columns_list.empty()) - finishDataSerialization(checksums, sync); + fillDataChecksums(checksums); if (settings.rewrite_primary_key) - finishPrimaryIndexSerialization(checksums, sync); + fillPrimaryIndexChecksums(checksums); - finishSkipIndicesSerialization(checksums, sync); + fillSkipIndicesChecksums(checksums); +} + +void MergeTreeDataPartWriterWide::finish(bool sync) +{ + // If we don't have anything to write, skip finalization. + if (!columns_list.empty()) + finishDataSerialization(sync); + + if (settings.rewrite_primary_key) + finishPrimaryIndexSerialization(sync); + + finishSkipIndicesSerialization(sync); } void MergeTreeDataPartWriterWide::writeFinalMark( diff --git a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h index 6303fbbac0d..b82fcd652ae 100644 --- a/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h +++ b/src/Storages/MergeTree/MergeTreeDataPartWriterWide.h @@ -29,12 +29,15 @@ public: void write(const Block & block, const IColumn::Permutation * permutation) override; - void finish(IMergeTreeDataPart::Checksums & checksums, bool sync) final; + void fillChecksums(IMergeTreeDataPart::Checksums & checksums) final; + + void finish(bool sync) final; private: /// Finish serialization of data: write final mark if required and compute checksums /// Also validate written data in debug mode - void finishDataSerialization(IMergeTreeDataPart::Checksums & checksums, bool sync); + void fillDataChecksums(IMergeTreeDataPart::Checksums & checksums); + void finishDataSerialization(bool sync); /// Write data of one column. /// Return how many marks were written and diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index cdedd37e14a..646b672a1e9 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -1,5 +1,4 @@ #include /// For calculations related to sampling coefficients. -#include #include #include @@ -988,9 +987,8 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd for (size_t part_index = 0; part_index < parts.size(); ++part_index) pool.scheduleOrThrowOnError([&, part_index, thread_group = CurrentThread::getGroup()] { - SCOPE_EXIT_SAFE(if (thread_group) CurrentThread::detachQueryIfNotDetached();); if (thread_group) - CurrentThread::attachTo(thread_group); + CurrentThread::attachToIfDetached(thread_group); process_part(part_index); }); @@ -1528,10 +1526,19 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( size_t final_mark = part->index_granularity.hasFinalMark(); size_t index_marks_count = (marks_count - final_mark + index_granularity - 1) / index_granularity; + MarkRanges index_ranges; + for (const auto & range : ranges) + { + MarkRange index_range( + range.begin / index_granularity, + (range.end + index_granularity - 1) / index_granularity); + index_ranges.push_back(index_range); + } + MergeTreeIndexReader reader( index_helper, part, index_marks_count, - ranges, + index_ranges, mark_cache, uncompressed_cache, reader_settings); @@ -1542,11 +1549,9 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( /// this variable is stored to avoid reading the same granule twice. MergeTreeIndexGranulePtr granule = nullptr; size_t last_index_mark = 0; - for (const auto & range : ranges) + for (size_t i = 0; i < ranges.size(); ++i) { - MarkRange index_range( - range.begin / index_granularity, - (range.end + index_granularity - 1) / index_granularity); + const MarkRange & index_range = index_ranges[i]; if (last_index_mark != index_range.begin || !granule) reader.seek(index_range.begin); @@ -1559,8 +1564,8 @@ MarkRanges MergeTreeDataSelectExecutor::filterMarksUsingIndex( granule = reader.read(); MarkRange data_range( - std::max(range.begin, index_mark * index_granularity), - std::min(range.end, (index_mark + 1) * index_granularity)); + std::max(ranges[i].begin, index_mark * index_granularity), + std::min(ranges[i].end, (index_mark + 1) * index_granularity)); if (!condition->mayBeTrueOnGranule(granule)) { diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 825fea38272..d16b5274a45 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -137,6 +137,12 @@ void updateTTL( } +void MergeTreeDataWriter::TemporaryPart::finalize() +{ + for (auto & stream : streams) + stream.finalizer.finish(); +} + BlocksWithPartition MergeTreeDataWriter::splitBlockIntoParts( const Block & block, size_t max_parts, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { @@ -270,9 +276,10 @@ Block MergeTreeDataWriter::mergeBlock( return block.cloneWithColumns(status.chunk.getColumns()); } -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempPart( BlockWithPartition & block_with_partition, const StorageMetadataPtr & metadata_snapshot, ContextPtr context) { + TemporaryPart temp_part; Block & block = block_with_partition.block; static const String TMP_PREFIX = "tmp_insert_"; @@ -343,7 +350,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart( /// If optimize_on_insert is true, block may become empty after merge. /// There is no need to create empty part. if (expected_size == 0) - return nullptr; + return temp_part; DB::IMergeTreeDataPart::TTLInfos move_ttl_infos; const auto & move_ttl_entries = metadata_snapshot->getMoveTTLs(); @@ -419,30 +426,37 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempPart( auto compression_codec = data.getContext()->chooseCompressionCodec(0, 0); const auto & index_factory = MergeTreeIndexFactory::instance(); - MergedBlockOutputStream out(new_data_part, metadata_snapshot,columns, + auto out = std::make_unique(new_data_part, metadata_snapshot,columns, index_factory.getMany(metadata_snapshot->getSecondaryIndices()), compression_codec); - bool sync_on_insert = data_settings->fsync_after_insert; - - out.writeWithPermutation(block, perm_ptr); + out->writeWithPermutation(block, perm_ptr); for (const auto & projection : metadata_snapshot->getProjections()) { auto projection_block = projection.calculate(block, context); if (projection_block.rows()) - new_data_part->addProjectionPart( - projection.name, writeProjectionPart(data, log, projection_block, projection, new_data_part.get())); + { + auto proj_temp_part = writeProjectionPart(data, log, projection_block, projection, new_data_part.get()); + new_data_part->addProjectionPart(projection.name, std::move(proj_temp_part.part)); + for (auto & stream : proj_temp_part.streams) + temp_part.streams.emplace_back(std::move(stream)); + } } - out.writeSuffixAndFinalizePart(new_data_part, sync_on_insert); + auto finalizer = out->finalizePartAsync(new_data_part, data_settings->fsync_after_insert); + + temp_part.part = new_data_part; + temp_part.streams.emplace_back(TemporaryPart::Stream{.stream = std::move(out), .finalizer = std::move(finalizer)}); + + /// out.finish(new_data_part, std::move(written_files), sync_on_insert); ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterRows, block.rows()); ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterUncompressedBytes, block.bytes()); ProfileEvents::increment(ProfileEvents::MergeTreeDataWriterCompressedBytes, new_data_part->getBytesOnDisk()); - return new_data_part; + return temp_part; } -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPartImpl( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPartImpl( const String & part_name, MergeTreeDataPartType part_type, const String & relative_path, @@ -453,6 +467,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPartImpl( Block block, const ProjectionDescription & projection) { + TemporaryPart temp_part; const StorageMetadataPtr & metadata_snapshot = projection.metadata; MergeTreePartInfo new_part_info("all", 0, 0, 0); auto new_data_part = data.createPart( @@ -525,24 +540,28 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPartImpl( /// either default lz4 or compression method with zero thresholds on absolute and relative part size. auto compression_codec = data.getContext()->chooseCompressionCodec(0, 0); - MergedBlockOutputStream out( + auto out = std::make_unique( new_data_part, metadata_snapshot, columns, - {}, + MergeTreeIndices{}, compression_codec); - out.writeWithPermutation(block, perm_ptr); - out.writeSuffixAndFinalizePart(new_data_part); + out->writeWithPermutation(block, perm_ptr); + auto finalizer = out->finalizePartAsync(new_data_part, false); + temp_part.part = new_data_part; + temp_part.streams.emplace_back(TemporaryPart::Stream{.stream = std::move(out), .finalizer = std::move(finalizer)}); + + // out.finish(new_data_part, std::move(written_files), false); ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterRows, block.rows()); ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterUncompressedBytes, block.bytes()); ProfileEvents::increment(ProfileEvents::MergeTreeDataProjectionWriterCompressedBytes, new_data_part->getBytesOnDisk()); - return new_data_part; + return temp_part; } -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPart( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeProjectionPart( MergeTreeData & data, Poco::Logger * log, Block block, const ProjectionDescription & projection, const IMergeTreeDataPart * parent_part) { String part_name = projection.name; @@ -574,7 +593,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeProjectionPart( /// This is used for projection materialization process which may contain multiple stages of /// projection part merges. -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempProjectionPart( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeTempProjectionPart( MergeTreeData & data, Poco::Logger * log, Block block, @@ -609,7 +628,7 @@ MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeTempProjectionPart( projection); } -MergeTreeData::MutableDataPartPtr MergeTreeDataWriter::writeInMemoryProjectionPart( +MergeTreeDataWriter::TemporaryPart MergeTreeDataWriter::writeInMemoryProjectionPart( const MergeTreeData & data, Poco::Logger * log, Block block, diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.h b/src/Storages/MergeTree/MergeTreeDataWriter.h index f16ec877113..ae46a94ccd7 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.h +++ b/src/Storages/MergeTree/MergeTreeDataWriter.h @@ -10,6 +10,7 @@ #include #include +#include namespace DB @@ -46,11 +47,28 @@ public: */ MergeTreeData::MutableDataPartPtr writeTempPart(BlockWithPartition & block, const StorageMetadataPtr & metadata_snapshot, bool optimize_on_insert); - MergeTreeData::MutableDataPartPtr - writeTempPart(BlockWithPartition & block, const StorageMetadataPtr & metadata_snapshot, ContextPtr context); + /// This structure contains not completely written temporary part. + /// Some writes may happen asynchronously, e.g. for blob storages. + /// You should call finalize() to wait until all data is written. + struct TemporaryPart + { + MergeTreeData::MutableDataPartPtr part; + + struct Stream + { + std::unique_ptr stream; + MergedBlockOutputStream::Finalizer finalizer; + }; + + std::vector streams; + + void finalize(); + }; + + TemporaryPart writeTempPart(BlockWithPartition & block, const StorageMetadataPtr & metadata_snapshot, ContextPtr context); /// For insertion. - static MergeTreeData::MutableDataPartPtr writeProjectionPart( + static TemporaryPart writeProjectionPart( MergeTreeData & data, Poco::Logger * log, Block block, @@ -58,7 +76,7 @@ public: const IMergeTreeDataPart * parent_part); /// For mutation: MATERIALIZE PROJECTION. - static MergeTreeData::MutableDataPartPtr writeTempProjectionPart( + static TemporaryPart writeTempProjectionPart( MergeTreeData & data, Poco::Logger * log, Block block, @@ -67,7 +85,7 @@ public: size_t block_num); /// For WriteAheadLog AddPart. - static MergeTreeData::MutableDataPartPtr writeInMemoryProjectionPart( + static TemporaryPart writeInMemoryProjectionPart( const MergeTreeData & data, Poco::Logger * log, Block block, @@ -82,7 +100,7 @@ public: const MergeTreeData::MergingParams & merging_params); private: - static MergeTreeData::MutableDataPartPtr writeProjectionPartImpl( + static TemporaryPart writeProjectionPartImpl( const String & part_name, MergeTreeDataPartType part_type, const String & relative_path, diff --git a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h index 467292d88bb..feacc159d7e 100644 --- a/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeInOrderSelectProcessor.h @@ -15,7 +15,7 @@ public: explicit MergeTreeInOrderSelectProcessor(Args &&... args) : MergeTreeSelectProcessor{std::forward(args)...} { - LOG_DEBUG(log, "Reading {} ranges in order from part {}, approx. {} rows starting from {}", + LOG_TRACE(log, "Reading {} ranges in order from part {}, approx. {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); } diff --git a/src/Storages/MergeTree/MergeTreePartition.cpp b/src/Storages/MergeTree/MergeTreePartition.cpp index 5b3c93aa553..706d72771e9 100644 --- a/src/Storages/MergeTree/MergeTreePartition.cpp +++ b/src/Storages/MergeTree/MergeTreePartition.cpp @@ -375,17 +375,17 @@ void MergeTreePartition::load(const MergeTreeData & storage, const DiskPtr & dis partition_key_sample.getByPosition(i).type->getDefaultSerialization()->deserializeBinary(value[i], *file); } -void MergeTreePartition::store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const +std::unique_ptr MergeTreePartition::store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const { auto metadata_snapshot = storage.getInMemoryMetadataPtr(); const auto & partition_key_sample = adjustPartitionKey(metadata_snapshot, storage.getContext()).sample_block; - store(partition_key_sample, disk, part_path, checksums); + return store(partition_key_sample, disk, part_path, checksums); } -void MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const +std::unique_ptr MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const { if (!partition_key_sample) - return; + return nullptr; auto out = disk->writeFile(part_path + "partition.dat"); HashingWriteBuffer out_hashing(*out); @@ -395,7 +395,8 @@ void MergeTreePartition::store(const Block & partition_key_sample, const DiskPtr out_hashing.next(); checksums.files["partition.dat"].file_size = out_hashing.count(); checksums.files["partition.dat"].file_hash = out_hashing.getHash(); - out->finalize(); + out->preFinalize(); + return out; } void MergeTreePartition::create(const StorageMetadataPtr & metadata_snapshot, Block block, size_t row, ContextPtr context) diff --git a/src/Storages/MergeTree/MergeTreePartition.h b/src/Storages/MergeTree/MergeTreePartition.h index d501d615621..f149fcbcb7e 100644 --- a/src/Storages/MergeTree/MergeTreePartition.h +++ b/src/Storages/MergeTree/MergeTreePartition.h @@ -38,8 +38,10 @@ public: void serializeText(const MergeTreeData & storage, WriteBuffer & out, const FormatSettings & format_settings) const; void load(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path); - void store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; - void store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; + /// Store functions return write buffer with written but not finalized data. + /// User must call finish() for returned object. + [[nodiscard]] std::unique_ptr store(const MergeTreeData & storage, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; + [[nodiscard]] std::unique_ptr store(const Block & partition_key_sample, const DiskPtr & disk, const String & part_path, MergeTreeDataPartChecksums & checksums) const; void assign(const MergeTreePartition & other) { value = other.value; } diff --git a/src/Storages/MergeTree/MergeTreePartsMover.cpp b/src/Storages/MergeTree/MergeTreePartsMover.cpp index 190fc0d30a0..9cc3ffe6e9e 100644 --- a/src/Storages/MergeTree/MergeTreePartsMover.cpp +++ b/src/Storages/MergeTree/MergeTreePartsMover.cpp @@ -113,7 +113,7 @@ bool MergeTreePartsMover::selectPartsForMove( UInt64 required_maximum_available_space = disk->getTotalSpace() * policy->getMoveFactor(); UInt64 unreserved_space = disk->getUnreservedSpace(); - if (unreserved_space < required_maximum_available_space) + if (unreserved_space < required_maximum_available_space && !disk->isBroken()) need_to_move.emplace(disk, required_maximum_available_space - unreserved_space); } } @@ -211,7 +211,7 @@ MergeTreeData::DataPartPtr MergeTreePartsMover::clonePart(const MergeTreeMoveEnt String relative_path = part->relative_path; if (disk->exists(path_to_clone + relative_path)) { - LOG_WARNING(log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again."); + LOG_WARNING(log, "Path {} already exists. Will remove it and clone again.", fullPath(disk, path_to_clone + relative_path)); disk->removeRecursive(fs::path(path_to_clone) / relative_path / ""); } disk->createDirectories(path_to_clone); diff --git a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp index b594b59fdfa..6b265b9460c 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompact.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompact.cpp @@ -118,7 +118,7 @@ MergeTreeReaderCompact::MergeTreeReaderCompact( } catch (...) { - storage.reportBrokenPart(data_part->name); + storage.reportBrokenPart(data_part); throw; } } @@ -175,7 +175,7 @@ size_t MergeTreeReaderCompact::readRows( catch (Exception & e) { if (e.code() != ErrorCodes::MEMORY_LIMIT_EXCEEDED) - storage.reportBrokenPart(data_part->name); + storage.reportBrokenPart(data_part); /// Better diagnostics. e.addMessage("(while reading column " + column_from_part.name + ")"); @@ -183,7 +183,7 @@ size_t MergeTreeReaderCompact::readRows( } catch (...) { - storage.reportBrokenPart(data_part->name); + storage.reportBrokenPart(data_part); throw; } } @@ -277,8 +277,11 @@ void MergeTreeReaderCompact::seekToMark(size_t row_index, size_t column_index) void MergeTreeReaderCompact::adjustUpperBound(size_t last_mark) { - auto right_offset = marks_loader.getMark(last_mark).offset_in_compressed_file; - if (!right_offset) + size_t right_offset = 0; + if (last_mark < data_part->getMarksCount()) /// Otherwise read until the end of file + right_offset = marks_loader.getMark(last_mark).offset_in_compressed_file; + + if (right_offset == 0) { /// If already reading till the end of file. if (last_right_offset && *last_right_offset == 0) diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index c25b2104841..117ef43ecb2 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -42,7 +42,7 @@ MergeTreeReaderStream::MergeTreeReaderStream( { size_t left_mark = mark_range.begin; size_t right_mark = mark_range.end; - auto [right_offset, mark_range_bytes] = getRightOffsetAndBytesRange(left_mark, right_mark); + auto [_, mark_range_bytes] = getRightOffsetAndBytesRange(left_mark, right_mark); max_mark_range_bytes = std::max(max_mark_range_bytes, mark_range_bytes); sum_mark_range_bytes += mark_range_bytes; @@ -106,42 +106,88 @@ MergeTreeReaderStream::MergeTreeReaderStream( } -std::pair MergeTreeReaderStream::getRightOffsetAndBytesRange(size_t left_mark, size_t right_mark) +std::pair MergeTreeReaderStream::getRightOffsetAndBytesRange(size_t left_mark, size_t right_mark_non_included) { /// NOTE: if we are reading the whole file, then right_mark == marks_count /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. - /// If the end of range is inside the block, we will need to read it too. - size_t result_right_mark = right_mark; - if (right_mark < marks_count && marks_loader.getMark(right_mark).offset_in_decompressed_block > 0) + /// Special case, can happen in Collapsing/Replacing engines + if (marks_count == 0) + return std::make_pair(0, 0); + + assert(left_mark < marks_count); + assert(right_mark_non_included <= marks_count); + assert(left_mark <= right_mark_non_included); + + size_t result_right_offset; + if (0 < right_mark_non_included && right_mark_non_included < marks_count) { - auto indices = collections::range(right_mark, marks_count); - auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, [this](size_t i, size_t j) + auto right_mark = marks_loader.getMark(right_mark_non_included); + result_right_offset = right_mark.offset_in_compressed_file; + + bool need_to_check_marks_from_the_right = false; + + /// If the end of range is inside the block, we will need to read it too. + if (right_mark.offset_in_decompressed_block > 0) { - return marks_loader.getMark(i).offset_in_compressed_file < marks_loader.getMark(j).offset_in_compressed_file; - }); + need_to_check_marks_from_the_right = true; + } + else + { + size_t right_mark_included = right_mark_non_included - 1; + const MarkInCompressedFile & right_mark_included_in_file = marks_loader.getMark(right_mark_included); - result_right_mark = (it == indices.end() ? marks_count : *it); + /// Also, in LowCardinality dictionary several consecutive marks can point to + /// the same offset. So to get true bytes offset we have to get first + /// non-equal mark. + /// Example: + /// Mark 186, points to [2003111, 0] + /// Mark 187, points to [2003111, 0] + /// Mark 188, points to [2003111, 0] <--- for example need to read until 188 + /// Mark 189, points to [2003111, 0] <--- not suitable, because have same offset + /// Mark 190, points to [2003111, 0] + /// Mark 191, points to [2003111, 0] + /// Mark 192, points to [2081424, 0] <--- what we are looking for + /// Mark 193, points to [2081424, 0] + /// Mark 194, points to [2081424, 0] + if (right_mark_included_in_file.offset_in_compressed_file == result_right_offset) + need_to_check_marks_from_the_right = true; + } + + /// Let's go to the right and find mark with bigger offset in compressed file + if (need_to_check_marks_from_the_right) + { + bool found_bigger_mark = false; + for (size_t i = right_mark_non_included + 1; i < marks_count; ++i) + { + const auto & candidate_mark = marks_loader.getMark(i); + if (result_right_offset < candidate_mark.offset_in_compressed_file) + { + result_right_offset = candidate_mark.offset_in_compressed_file; + found_bigger_mark = true; + break; + } + } + + if (!found_bigger_mark) + { + /// If there are no marks after the end of range, just use file size + result_right_offset = file_size; + } + } } - - size_t right_offset; - size_t mark_range_bytes; - - /// If there are no marks after the end of range, just use file size - if (result_right_mark >= marks_count - || (result_right_mark + 1 == marks_count - && marks_loader.getMark(result_right_mark).offset_in_compressed_file == marks_loader.getMark(right_mark).offset_in_compressed_file)) + else if (right_mark_non_included == 0) { - right_offset = file_size; - mark_range_bytes = right_offset - (left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0); + result_right_offset = marks_loader.getMark(right_mark_non_included).offset_in_compressed_file; } else { - right_offset = marks_loader.getMark(result_right_mark).offset_in_compressed_file; - mark_range_bytes = right_offset - marks_loader.getMark(left_mark).offset_in_compressed_file; + result_right_offset = file_size; } - return std::make_pair(right_offset, mark_range_bytes); + size_t mark_range_bytes = result_right_offset - (left_mark < marks_count ? marks_loader.getMark(left_mark).offset_in_compressed_file : 0); + + return std::make_pair(result_right_offset, mark_range_bytes); } @@ -197,7 +243,7 @@ void MergeTreeReaderStream::adjustForRange(MarkRange range) * read from stream, but we must update last_right_offset only if it is bigger than * the last one to avoid redundantly cancelling prefetches. */ - auto [right_offset, mark_range_bytes] = getRightOffsetAndBytesRange(range.begin, range.end); + auto [right_offset, _] = getRightOffsetAndBytesRange(range.begin, range.end); if (!right_offset) { if (last_right_offset && *last_right_offset == 0) diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index b1410953ddb..96f3ea31fec 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -39,7 +39,7 @@ public: ReadBuffer * data_buffer; private: - std::pair getRightOffsetAndBytesRange(size_t left_mark, size_t right_mark); + std::pair getRightOffsetAndBytesRange(size_t left_mark, size_t right_mark_non_included); DiskPtr disk; std::string path_prefix; diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 5e51a2931e4..77c0e8332d4 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -56,7 +56,7 @@ MergeTreeReaderWide::MergeTreeReaderWide( } catch (...) { - storage.reportBrokenPart(data_part->name); + storage.reportBrokenPart(data_part); throw; } } @@ -144,7 +144,7 @@ size_t MergeTreeReaderWide::readRows( catch (Exception & e) { if (e.code() != ErrorCodes::MEMORY_LIMIT_EXCEEDED) - storage.reportBrokenPart(data_part->name); + storage.reportBrokenPart(data_part); /// Better diagnostics. e.addMessage("(while reading from part " + data_part->getFullPath() + " " @@ -154,7 +154,7 @@ size_t MergeTreeReaderWide::readRows( } catch (...) { - storage.reportBrokenPart(data_part->name); + storage.reportBrokenPart(data_part); throw; } diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp index 6c4059d64d0..1a2ab8bff5b 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.cpp @@ -40,7 +40,7 @@ catch (...) { /// Suspicion of the broken part. A part is added to the queue for verification. if (getCurrentExceptionCode() != ErrorCodes::MEMORY_LIMIT_EXCEEDED) - storage.reportBrokenPart(data_part->name); + storage.reportBrokenPart(data_part); throw; } diff --git a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h index 395f5d5cd2a..38dcc1a2352 100644 --- a/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeReverseSelectProcessor.h @@ -16,7 +16,7 @@ public: explicit MergeTreeReverseSelectProcessor(Args &&... args) : MergeTreeSelectProcessor{std::forward(args)...} { - LOG_DEBUG(log, "Reading {} ranges in reverse order from part {}, approx. {} rows starting from {}", + LOG_TRACE(log, "Reading {} ranges in reverse order from part {}, approx. {} rows starting from {}", all_mark_ranges.size(), data_part->name, total_rows, data_part->index_granularity.getMarkStartingRow(all_mark_ranges.front().begin)); } diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 687458ee681..37012aa6570 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -80,8 +80,7 @@ try const auto & sample = reader->getColumns(); Columns columns(sample.size()); - /// TODO: pass stream size instead of zero? - size_t rows_read = reader->readRows(current_mark, 0, continue_reading, rows_to_read, columns); + size_t rows_read = reader->readRows(current_mark, data_part->getMarksCount(), continue_reading, rows_to_read, columns); if (rows_read) { @@ -126,7 +125,7 @@ catch (...) { /// Suspicion of the broken part. A part is added to the queue for verification. if (getCurrentExceptionCode() != ErrorCodes::MEMORY_LIMIT_EXCEEDED) - storage.reportBrokenPart(data_part->name); + storage.reportBrokenPart(data_part); throw; } diff --git a/src/Storages/MergeTree/MergeTreeSink.cpp b/src/Storages/MergeTree/MergeTreeSink.cpp index 5e97f80d849..ea252954b7e 100644 --- a/src/Storages/MergeTree/MergeTreeSink.cpp +++ b/src/Storages/MergeTree/MergeTreeSink.cpp @@ -7,6 +7,21 @@ namespace DB { +MergeTreeSink::~MergeTreeSink() = default; + +MergeTreeSink::MergeTreeSink( + StorageMergeTree & storage_, + StorageMetadataPtr metadata_snapshot_, + size_t max_parts_per_block_, + ContextPtr context_) + : SinkToStorage(metadata_snapshot_->getSampleBlock()) + , storage(storage_) + , metadata_snapshot(metadata_snapshot_) + , max_parts_per_block(max_parts_per_block_) + , context(context_) +{ +} + void MergeTreeSink::onStart() { /// Only check "too many parts" before write, @@ -14,32 +29,91 @@ void MergeTreeSink::onStart() storage.delayInsertOrThrowIfNeeded(); } +void MergeTreeSink::onFinish() +{ + finishDelayedChunk(); +} + +struct MergeTreeSink::DelayedChunk +{ + struct Partition + { + MergeTreeDataWriter::TemporaryPart temp_part; + UInt64 elapsed_ns; + String block_dedup_token; + }; + + std::vector partitions; +}; + void MergeTreeSink::consume(Chunk chunk) { auto block = getHeader().cloneWithColumns(chunk.detachColumns()); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); + std::vector partitions; for (auto & current_block : part_blocks) { Stopwatch watch; + String block_dedup_token; - MergeTreeData::MutableDataPartPtr part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); + auto temp_part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); + + UInt64 elapsed_ns = watch.elapsed(); /// If optimize_on_insert setting is true, current_block could become empty after merge /// and we didn't create part. - if (!part) + if (!temp_part.part) continue; - /// Part can be deduplicated, so increment counters and add to part log only if it's really added - if (storage.renameTempPartAndAdd(part, &storage.increment, nullptr, storage.getDeduplicationLog())) + if (storage.getDeduplicationLog()) { - PartLog::addNewPart(storage.getContext(), part, watch.elapsed()); + const String & dedup_token = context->getSettingsRef().insert_deduplication_token; + if (!dedup_token.empty()) + { + /// multiple blocks can be inserted within the same insert query + /// an ordinal number is added to dedup token to generate a distinctive block id for each block + block_dedup_token = fmt::format("{}_{}", dedup_token, chunk_dedup_seqnum); + ++chunk_dedup_seqnum; + } + } + + partitions.emplace_back(MergeTreeSink::DelayedChunk::Partition + { + .temp_part = std::move(temp_part), + .elapsed_ns = elapsed_ns, + .block_dedup_token = std::move(block_dedup_token) + }); + } + + finishDelayedChunk(); + delayed_chunk = std::make_unique(); + delayed_chunk->partitions = std::move(partitions); +} + +void MergeTreeSink::finishDelayedChunk() +{ + if (!delayed_chunk) + return; + + for (auto & partition : delayed_chunk->partitions) + { + partition.temp_part.finalize(); + + auto & part = partition.temp_part.part; + + /// Part can be deduplicated, so increment counters and add to part log only if it's really added + if (storage.renameTempPartAndAdd(part, &storage.increment, nullptr, storage.getDeduplicationLog(), partition.block_dedup_token)) + { + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns); /// Initiate async merge - it will be done if it's good time for merge and if there are space in 'background_pool'. storage.background_operations_assignee.trigger(); } } + + delayed_chunk.reset(); } } diff --git a/src/Storages/MergeTree/MergeTreeSink.h b/src/Storages/MergeTree/MergeTreeSink.h index 60ac62c7592..65a565d7f57 100644 --- a/src/Storages/MergeTree/MergeTreeSink.h +++ b/src/Storages/MergeTree/MergeTreeSink.h @@ -16,26 +16,29 @@ class MergeTreeSink : public SinkToStorage public: MergeTreeSink( StorageMergeTree & storage_, - const StorageMetadataPtr metadata_snapshot_, + StorageMetadataPtr metadata_snapshot_, size_t max_parts_per_block_, - ContextPtr context_) - : SinkToStorage(metadata_snapshot_->getSampleBlock()) - , storage(storage_) - , metadata_snapshot(metadata_snapshot_) - , max_parts_per_block(max_parts_per_block_) - , context(context_) - { - } + ContextPtr context_); + + ~MergeTreeSink() override; String getName() const override { return "MergeTreeSink"; } void consume(Chunk chunk) override; void onStart() override; + void onFinish() override; private: StorageMergeTree & storage; StorageMetadataPtr metadata_snapshot; size_t max_parts_per_block; ContextPtr context; + uint64_t chunk_dedup_seqnum = 0; /// input chunk ordinal number in case of dedup token + + /// We can delay processing for previous chunk and start writing a new one. + struct DelayedChunk; + std::unique_ptr delayed_chunk; + + void finishDelayedChunk(); }; } diff --git a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp index 694357ab0c2..d7cddfe9c14 100644 --- a/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp +++ b/src/Storages/MergeTree/MergeTreeWriteAheadLog.cpp @@ -109,6 +109,8 @@ void MergeTreeWriteAheadLog::rotate(const std::unique_lock &) + toString(min_block_number) + "_" + toString(max_block_number) + WAL_FILE_EXTENSION; + /// Finalize stream before file rename + out->finalize(); disk->replaceFile(path, storage.getRelativeDataPath() + new_name); init(); } @@ -208,12 +210,12 @@ MergeTreeData::MutableDataPartsVector MergeTreeWriteAheadLog::restore(const Stor for (const auto & projection : metadata_snapshot->getProjections()) { auto projection_block = projection.calculate(block, context); + auto temp_part = MergeTreeDataWriter::writeInMemoryProjectionPart(storage, log, projection_block, projection, part.get()); + temp_part.finalize(); if (projection_block.rows()) - part->addProjectionPart( - projection.name, - MergeTreeDataWriter::writeInMemoryProjectionPart(storage, log, projection_block, projection, part.get())); + part->addProjectionPart(projection.name, std::move(temp_part.part)); } - part_out.writeSuffixAndFinalizePart(part); + part_out.finalizePart(part, false); min_block_number = std::min(min_block_number, part->info.min_block); max_block_number = std::max(max_block_number, part->info.max_block); diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.cpp b/src/Storages/MergeTree/MergedBlockOutputStream.cpp index 5274118df29..b525285979e 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.cpp +++ b/src/Storages/MergeTree/MergedBlockOutputStream.cpp @@ -51,7 +51,74 @@ void MergedBlockOutputStream::writeWithPermutation(const Block & block, const IC writeImpl(block, permutation); } -void MergedBlockOutputStream::writeSuffixAndFinalizePart( +struct MergedBlockOutputStream::Finalizer::Impl +{ + IMergeTreeDataPartWriter & writer; + MergeTreeData::MutableDataPartPtr part; + NameSet files_to_remove_after_finish; + std::vector> written_files; + bool sync; + + Impl(IMergeTreeDataPartWriter & writer_, MergeTreeData::MutableDataPartPtr part_, const NameSet & files_to_remove_after_finish_, bool sync_) + : writer(writer_) + , part(std::move(part_)) + , files_to_remove_after_finish(files_to_remove_after_finish_) + , sync(sync_) {} + + void finish(); +}; + +void MergedBlockOutputStream::Finalizer::finish() +{ + std::unique_ptr to_finish = std::move(impl); + if (to_finish) + to_finish->finish(); +} + +void MergedBlockOutputStream::Finalizer::Impl::finish() +{ + writer.finish(sync); + + auto disk = part->volume->getDisk(); + for (const auto & file_name: files_to_remove_after_finish) + disk->removeFile(part->getFullRelativePath() + file_name); + + for (auto & file : written_files) + { + file->finalize(); + if (sync) + file->sync(); + } + + part->storage.lockSharedData(*part); +} + +MergedBlockOutputStream::Finalizer::~Finalizer() +{ + try + { + finish(); + } + catch (...) + { + tryLogCurrentException("MergedBlockOutputStream"); + } +} + +MergedBlockOutputStream::Finalizer::Finalizer(Finalizer &&) = default; +MergedBlockOutputStream::Finalizer & MergedBlockOutputStream::Finalizer::operator=(Finalizer &&) = default; +MergedBlockOutputStream::Finalizer::Finalizer(std::unique_ptr impl_) : impl(std::move(impl_)) {} + +void MergedBlockOutputStream::finalizePart( + MergeTreeData::MutableDataPartPtr & new_part, + bool sync, + const NamesAndTypesList * total_columns_list, + MergeTreeData::DataPart::Checksums * additional_column_checksums) +{ + finalizePartAsync(new_part, sync, total_columns_list, additional_column_checksums).finish(); +} + +MergedBlockOutputStream::Finalizer MergedBlockOutputStream::finalizePartAsync( MergeTreeData::MutableDataPartPtr & new_part, bool sync, const NamesAndTypesList * total_columns_list, @@ -64,7 +131,9 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( checksums = std::move(*additional_column_checksums); /// Finish columns serialization. - writer->finish(checksums, sync); + writer->fillChecksums(checksums); + + LOG_TRACE(&Poco::Logger::get("MergedBlockOutputStream"), "filled checksums {}", new_part->getNameWithState()); for (const auto & [projection_name, projection_part] : new_part->getProjectionParts()) checksums.addFile( @@ -72,20 +141,22 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( projection_part->checksums.getTotalSizeOnDisk(), projection_part->checksums.getTotalChecksumUInt128()); + NameSet files_to_remove_after_sync; if (reset_columns) { auto part_columns = total_columns_list ? *total_columns_list : columns_list; auto serialization_infos = new_part->getSerializationInfos(); serialization_infos.replaceData(new_serialization_infos); - removeEmptyColumnsFromPart(new_part, part_columns, serialization_infos, checksums); + files_to_remove_after_sync = removeEmptyColumnsFromPart(new_part, part_columns, serialization_infos, checksums); new_part->setColumns(part_columns); new_part->setSerializationInfos(serialization_infos); } + auto finalizer = std::make_unique(*writer, new_part, files_to_remove_after_sync, sync); if (new_part->isStoredOnDisk()) - finalizePartOnDisk(new_part, checksums, sync); + finalizer->written_files = finalizePartOnDisk(new_part, checksums); new_part->rows_count = rows_count; new_part->modification_time = time(nullptr); @@ -97,15 +168,15 @@ void MergedBlockOutputStream::writeSuffixAndFinalizePart( if (default_codec != nullptr) new_part->default_codec = default_codec; - new_part->storage.lockSharedData(*new_part); + + return Finalizer(std::move(finalizer)); } -void MergedBlockOutputStream::finalizePartOnDisk( +MergedBlockOutputStream::WrittenFiles MergedBlockOutputStream::finalizePartOnDisk( const MergeTreeData::DataPartPtr & new_part, - MergeTreeData::DataPart::Checksums & checksums, - bool sync) + MergeTreeData::DataPart::Checksums & checksums) { - + WrittenFiles written_files; if (new_part->isProjectionPart()) { if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING || isCompactPart(new_part)) @@ -116,6 +187,8 @@ void MergedBlockOutputStream::finalizePartOnDisk( count_out_hashing.next(); checksums.files["count.txt"].file_size = count_out_hashing.count(); checksums.files["count.txt"].file_hash = count_out_hashing.getHash(); + count_out->preFinalize(); + written_files.emplace_back(std::move(count_out)); } } else @@ -127,16 +200,21 @@ void MergedBlockOutputStream::finalizePartOnDisk( writeUUIDText(new_part->uuid, out_hashing); checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_size = out_hashing.count(); checksums.files[IMergeTreeDataPart::UUID_FILE_NAME].file_hash = out_hashing.getHash(); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } if (storage.format_version >= MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { - new_part->partition.store(storage, volume->getDisk(), part_path, checksums); + if (auto file = new_part->partition.store(storage, volume->getDisk(), part_path, checksums)) + written_files.emplace_back(std::move(file)); + if (new_part->minmax_idx->initialized) - new_part->minmax_idx->store(storage, volume->getDisk(), part_path, checksums); + { + auto files = new_part->minmax_idx->store(storage, volume->getDisk(), part_path, checksums); + for (auto & file : files) + written_files.emplace_back(std::move(file)); + } else if (rows_count) throw Exception("MinMax index was not initialized for new non-empty part " + new_part->name + ". It is a bug.", ErrorCodes::LOGICAL_ERROR); @@ -149,9 +227,8 @@ void MergedBlockOutputStream::finalizePartOnDisk( count_out_hashing.next(); checksums.files["count.txt"].file_size = count_out_hashing.count(); checksums.files["count.txt"].file_hash = count_out_hashing.getHash(); - count_out->finalize(); - if (sync) - count_out->sync(); + count_out->preFinalize(); + written_files.emplace_back(std::move(count_out)); } } @@ -163,9 +240,8 @@ void MergedBlockOutputStream::finalizePartOnDisk( new_part->ttl_infos.write(out_hashing); checksums.files["ttl.txt"].file_size = out_hashing.count(); checksums.files["ttl.txt"].file_hash = out_hashing.getHash(); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } if (!new_part->getSerializationInfos().empty()) @@ -175,25 +251,24 @@ void MergedBlockOutputStream::finalizePartOnDisk( new_part->getSerializationInfos().writeJSON(out_hashing); checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_size = out_hashing.count(); checksums.files[IMergeTreeDataPart::SERIALIZATION_FILE_NAME].file_hash = out_hashing.getHash(); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } { /// Write a file with a description of columns. auto out = volume->getDisk()->writeFile(fs::path(part_path) / "columns.txt", 4096); new_part->getColumns().writeText(*out); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } if (default_codec != nullptr) { auto out = volume->getDisk()->writeFile(part_path + IMergeTreeDataPart::DEFAULT_COMPRESSION_CODEC_FILE_NAME, 4096); DB::writeText(queryToString(default_codec->getFullCodecDesc()), *out); - out->finalize(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } else { @@ -205,10 +280,11 @@ void MergedBlockOutputStream::finalizePartOnDisk( /// Write file with checksums. auto out = volume->getDisk()->writeFile(fs::path(part_path) / "checksums.txt", 4096); checksums.write(*out); - out->finalize(); - if (sync) - out->sync(); + out->preFinalize(); + written_files.emplace_back(std::move(out)); } + + return written_files; } void MergedBlockOutputStream::writeImpl(const Block & block, const IColumn::Permutation * permutation) diff --git a/src/Storages/MergeTree/MergedBlockOutputStream.h b/src/Storages/MergeTree/MergedBlockOutputStream.h index 21e3c794239..b38395d56c2 100644 --- a/src/Storages/MergeTree/MergedBlockOutputStream.h +++ b/src/Storages/MergeTree/MergedBlockOutputStream.h @@ -32,11 +32,33 @@ public: */ void writeWithPermutation(const Block & block, const IColumn::Permutation * permutation); + /// Finalizer is a structure which is returned from by finalizePart(). + /// Files from part may be written asynchronously, e.g. for blob storages. + /// You should call finish() to wait until all data is written. + struct Finalizer + { + struct Impl; + std::unique_ptr impl; + + explicit Finalizer(std::unique_ptr impl_); + ~Finalizer(); + Finalizer(Finalizer &&); + Finalizer & operator=(Finalizer &&); + + void finish(); + }; + /// Finalize writing part and fill inner structures /// If part is new and contains projections, they should be added before invoking this method. - void writeSuffixAndFinalizePart( + Finalizer finalizePartAsync( MergeTreeData::MutableDataPartPtr & new_part, - bool sync = false, + bool sync, + const NamesAndTypesList * total_columns_list = nullptr, + MergeTreeData::DataPart::Checksums * additional_column_checksums = nullptr); + + void finalizePart( + MergeTreeData::MutableDataPartPtr & new_part, + bool sync, const NamesAndTypesList * total_columns_list = nullptr, MergeTreeData::DataPart::Checksums * additional_column_checksums = nullptr); @@ -46,10 +68,10 @@ private: */ void writeImpl(const Block & block, const IColumn::Permutation * permutation); - void finalizePartOnDisk( + using WrittenFiles = std::vector>; + WrittenFiles finalizePartOnDisk( const MergeTreeData::DataPartPtr & new_part, - MergeTreeData::DataPart::Checksums & checksums, - bool sync); + MergeTreeData::DataPart::Checksums & checksums); NamesAndTypesList columns_list; IMergeTreeDataPart::MinMaxIndex minmax_idx; diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp index 4c43e93e809..5a706165000 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.cpp @@ -55,14 +55,13 @@ void MergedColumnOnlyOutputStream::write(const Block & block) } MergeTreeData::DataPart::Checksums -MergedColumnOnlyOutputStream::writeSuffixAndGetChecksums( +MergedColumnOnlyOutputStream::fillChecksums( MergeTreeData::MutableDataPartPtr & new_part, - MergeTreeData::DataPart::Checksums & all_checksums, - bool sync) + MergeTreeData::DataPart::Checksums & all_checksums) { /// Finish columns serialization. MergeTreeData::DataPart::Checksums checksums; - writer->finish(checksums, sync); + writer->fillChecksums(checksums); for (const auto & [projection_name, projection_part] : new_part->getProjectionParts()) checksums.addFile( @@ -75,9 +74,18 @@ MergedColumnOnlyOutputStream::writeSuffixAndGetChecksums( serialization_infos.replaceData(new_serialization_infos); auto removed_files = removeEmptyColumnsFromPart(new_part, columns, serialization_infos, checksums); + + auto disk = new_part->volume->getDisk(); for (const String & removed_file : removed_files) + { + auto file_path = new_part->getFullRelativePath() + removed_file; + /// Can be called multiple times, don't need to remove file twice + if (disk->exists(file_path)) + disk->removeFile(file_path); + if (all_checksums.files.count(removed_file)) all_checksums.files.erase(removed_file); + } new_part->setColumns(columns); new_part->setSerializationInfos(serialization_infos); @@ -85,4 +93,9 @@ MergedColumnOnlyOutputStream::writeSuffixAndGetChecksums( return checksums; } +void MergedColumnOnlyOutputStream::finish(bool sync) +{ + writer->finish(sync); +} + } diff --git a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h index 4b75bc52f72..7b587d01dab 100644 --- a/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h +++ b/src/Storages/MergeTree/MergedColumnOnlyOutputStream.h @@ -25,8 +25,11 @@ public: Block getHeader() const { return header; } void write(const Block & block) override; + MergeTreeData::DataPart::Checksums - writeSuffixAndGetChecksums(MergeTreeData::MutableDataPartPtr & new_part, MergeTreeData::DataPart::Checksums & all_checksums, bool sync = false); + fillChecksums(MergeTreeData::MutableDataPartPtr & new_part, MergeTreeData::DataPart::Checksums & all_checksums); + + void finish(bool sync); private: Block header; diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 713f6a68612..4f1a663409d 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -28,9 +28,11 @@ std::pair MutateFromLogEntry if (source_part->name != source_part_name) { - LOG_WARNING(log, "Part " + source_part_name + " is covered by " + source_part->name - + " but should be mutated to " + entry.new_part_name + ". " - + "Possibly the mutation of this part is not needed and will be skipped. This shouldn't happen often."); + LOG_WARNING(log, + "Part {} is covered by {} but should be mutated to {}. " + "Possibly the mutation of this part is not needed and will be skipped. " + "This shouldn't happen often.", + source_part_name, source_part->name, entry.new_part_name); return {false, {}}; } @@ -50,6 +52,23 @@ std::pair MutateFromLogEntry } } + /// In some use cases merging can be more expensive than fetching + /// and it may be better to spread merges tasks across the replicas + /// instead of doing exactly the same merge cluster-wise + + if (storage.merge_strategy_picker.shouldMergeOnSingleReplica(entry)) + { + std::optional replica_to_execute_merge = storage.merge_strategy_picker.pickReplicaToExecuteMerge(entry); + if (replica_to_execute_merge) + { + LOG_DEBUG(log, + "Prefer fetching part {} from replica {} due to execute_merges_on_single_replica_time_threshold", + entry.new_part_name, replica_to_execute_merge.value()); + + return {false, {}}; + } + } + new_part_info = MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version); commands = MutationCommands::create(storage.queue.getMutationCommands(source_part, new_part_info.mutation)); @@ -71,6 +90,28 @@ std::pair MutateFromLogEntry future_mutated_part->updatePath(storage, reserved_space.get()); future_mutated_part->type = source_part->getType(); + if (storage_settings_ptr->allow_remote_fs_zero_copy_replication) + { + if (auto disk = reserved_space->getDisk(); disk->getType() == DB::DiskType::S3) + { + String dummy; + if (!storage.findReplicaHavingCoveringPart(entry.new_part_name, true, dummy).empty()) + { + LOG_DEBUG(log, "Mutation of part {} finished by some other replica, will download merged part", entry.new_part_name); + return {false, {}}; + } + + zero_copy_lock = storage.tryCreateZeroCopyExclusiveLock(entry.new_part_name, disk); + + if (!zero_copy_lock) + { + LOG_DEBUG(log, "Mutation of part {} started by some other replica, will wait it and fetch merged part", entry.new_part_name); + return {false, {}}; + } + } + } + + const Settings & settings = storage.getContext()->getSettingsRef(); merge_mutate_entry = storage.getContext()->getMergeList().insert( storage.getStorageID(), @@ -138,6 +179,12 @@ bool MutateFromLogEntryTask::finalize(ReplicatedMergeMutateTaskBase::PartLogWrit throw; } + if (zero_copy_lock) + { + LOG_DEBUG(log, "Removing zero-copy lock"); + zero_copy_lock->lock->unlock(); + } + /** With `ZSESSIONEXPIRED` or `ZOPERATIONTIMEOUT`, we can inadvertently roll back local changes to the parts. * This is not a problem, because in this case the entry will remain in the queue, and we will try again. */ diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.h b/src/Storages/MergeTree/MutateFromLogEntryTask.h index 5709e7b808a..c96fc9a8c49 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.h +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB { @@ -41,6 +42,7 @@ private: MutationCommandsConstPtr commands; MergeTreeData::TransactionUniquePtr transaction_ptr{nullptr}; + std::optional zero_copy_lock; StopwatchUniquePtr stopwatch_ptr{nullptr}; MergeTreeData::MutableDataPartPtr new_part{nullptr}; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 985098bb2a3..cc690287ef6 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -804,8 +804,12 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections() const auto & projection = *ctx->projections_to_build[i]; auto projection_block = projection_squashes[i].add(projection.calculate(cur_block, ctx->context)); if (projection_block) - projection_parts[projection.name].emplace_back(MergeTreeDataWriter::writeTempProjectionPart( - *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num)); + { + auto tmp_part = MergeTreeDataWriter::writeTempProjectionPart( + *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num); + tmp_part.finalize(); + projection_parts[projection.name].emplace_back(std::move(tmp_part.part)); + } } (*ctx->mutate_entry)->rows_written += cur_block.rows(); @@ -823,8 +827,10 @@ bool PartMergerWriter::mutateOriginalPartAndPrepareProjections() auto projection_block = projection_squash.add({}); if (projection_block) { - projection_parts[projection.name].emplace_back(MergeTreeDataWriter::writeTempProjectionPart( - *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num)); + auto temp_part = MergeTreeDataWriter::writeTempProjectionPart( + *ctx->data, ctx->log, projection_block, projection, ctx->new_data_part.get(), ++block_num); + temp_part.finalize(); + projection_parts[projection.name].emplace_back(std::move(temp_part.part)); } } @@ -976,7 +982,7 @@ private: ctx->mutating_executor.reset(); ctx->mutating_pipeline.reset(); - static_pointer_cast(ctx->out)->writeSuffixAndFinalizePart(ctx->new_data_part, ctx->need_sync); + static_pointer_cast(ctx->out)->finalizePart(ctx->new_data_part, ctx->need_sync); ctx->out.reset(); } @@ -1132,9 +1138,11 @@ private: ctx->mutating_pipeline.reset(); auto changed_checksums = - static_pointer_cast(ctx->out)->writeSuffixAndGetChecksums( - ctx->new_data_part, ctx->new_data_part->checksums, ctx->need_sync); + static_pointer_cast(ctx->out)->fillChecksums( + ctx->new_data_part, ctx->new_data_part->checksums); ctx->new_data_part->checksums.add(std::move(changed_checksums)); + + static_pointer_cast(ctx->out)->finish(ctx->need_sync); } for (const auto & [rename_from, rename_to] : ctx->files_to_rename) diff --git a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp index db5ca15ce8a..d73e77812c1 100644 --- a/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeMutateTaskBase.cpp @@ -31,6 +31,7 @@ bool ReplicatedMergeMutateTaskBase::executeStep() { std::exception_ptr saved_exception; + bool retryable_error = false; try { /// We don't have any backoff for failed entries @@ -45,17 +46,20 @@ bool ReplicatedMergeMutateTaskBase::executeStep() if (e.code() == ErrorCodes::NO_REPLICA_HAS_PART) { /// If no one has the right part, probably not all replicas work; We will not write to log with Error level. - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); + retryable_error = true; } else if (e.code() == ErrorCodes::ABORTED) { /// Interrupted merge or downloading a part is not an error. - LOG_INFO(log, e.message()); + LOG_INFO(log, fmt::runtime(e.message())); + retryable_error = true; } else if (e.code() == ErrorCodes::PART_IS_TEMPORARILY_LOCKED) { /// Part cannot be added temporarily - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); + retryable_error = true; storage.cleanup_thread.wakeup(); } else @@ -80,7 +84,7 @@ bool ReplicatedMergeMutateTaskBase::executeStep() } - if (saved_exception) + if (!retryable_error && saved_exception) { std::lock_guard lock(storage.queue.state_mutex); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index ff37a341205..26bfd951d3d 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -7,6 +7,8 @@ #include #include +#include + namespace DB { @@ -110,7 +112,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldLogs() if (entries.empty()) return; - std::sort(entries.begin(), entries.end()); + ::sort(entries.begin(), entries.end()); String min_saved_record_log_str = entries[ entries.size() > storage_settings->max_replicated_logs_to_keep @@ -443,7 +445,7 @@ void ReplicatedMergeTreeCleanupThread::getBlocksSortedByTime(zkutil::ZooKeeper & } } - std::sort(timed_blocks.begin(), timed_blocks.end(), NodeWithStat::greaterByTime); + ::sort(timed_blocks.begin(), timed_blocks.end(), NodeWithStat::greaterByTime); } @@ -476,7 +478,7 @@ void ReplicatedMergeTreeCleanupThread::clearOldMutations() } Strings entries = zookeeper->getChildren(storage.zookeeper_path + "/mutations"); - std::sort(entries.begin(), entries.end()); + ::sort(entries.begin(), entries.end()); /// Do not remove entries that are greater than `min_pointer` (they are not done yet). entries.erase(std::upper_bound(entries.begin(), entries.end(), padIndex(min_pointer)), entries.end()); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp index db931e771cf..44877c3da95 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.cpp @@ -2,14 +2,15 @@ #include #include - #include +#include #include #include #include #include #include + namespace DB { @@ -56,17 +57,6 @@ bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplica(const Re } -bool ReplicatedMergeTreeMergeStrategyPicker::shouldMergeOnSingleReplicaShared(const ReplicatedMergeTreeLogEntryData & entry) const -{ - time_t threshold = remote_fs_execute_merges_on_single_replica_time_threshold; - return ( - threshold > 0 /// feature turned on - && entry.type == ReplicatedMergeTreeLogEntry::MERGE_PARTS /// it is a merge log entry - && entry.create_time + threshold > time(nullptr) /// not too much time waited - ); -} - - /// that will return the same replica name for ReplicatedMergeTreeLogEntry on all the replicas (if the replica set is the same). /// that way each replica knows who is responsible for doing a certain merge. @@ -125,7 +115,7 @@ void ReplicatedMergeTreeMergeStrategyPicker::refreshState() auto zookeeper = storage.getZooKeeper(); auto all_replicas = zookeeper->getChildren(storage.zookeeper_path + "/replicas"); - std::sort(all_replicas.begin(), all_replicas.end()); + ::sort(all_replicas.begin(), all_replicas.end()); std::vector active_replicas_tmp; int current_replica_index_tmp = -1; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h index ca5ec601a99..70eacbee102 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeMergeStrategyPicker.h @@ -52,10 +52,6 @@ public: /// and we may need to do a fetch (or postpone) instead of merge bool shouldMergeOnSingleReplica(const ReplicatedMergeTreeLogEntryData & entry) const; - /// return true if remote_fs_execute_merges_on_single_replica_time_threshold feature is active - /// and we may need to do a fetch (or postpone) instead of merge - bool shouldMergeOnSingleReplicaShared(const ReplicatedMergeTreeLogEntryData & entry) const; - /// returns the replica name /// and it's not current replica should do the merge std::optional pickReplicaToExecuteMerge(const ReplicatedMergeTreeLogEntryData & entry); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp index 28b4cf306ec..af877bdbdf0 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreePartCheckThread.cpp @@ -359,7 +359,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na tryLogCurrentException(log, __PRETTY_FUNCTION__); String message = "Part " + part_name + " looks broken. Removing it and will try to fetch."; - LOG_ERROR(log, message); + LOG_ERROR(log, fmt::runtime(message)); /// Delete part locally. storage.forgetPartAndMoveToDetached(part, "broken"); @@ -378,7 +378,7 @@ CheckResult ReplicatedMergeTreePartCheckThread::checkPart(const String & part_na ProfileEvents::increment(ProfileEvents::ReplicatedPartChecksFailed); String message = "Unexpected part " + part_name + " in filesystem. Removing."; - LOG_ERROR(log, message); + LOG_ERROR(log, fmt::runtime(message)); storage.forgetPartAndMoveToDetached(part, "unexpected"); return {part_name, false, message}; } @@ -466,6 +466,8 @@ void ReplicatedMergeTreePartCheckThread::run() } } + storage.checkBrokenDisks(); + task->schedule(); } catch (const Coordination::Exception & e) diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp index 8b0751f4bbf..f13bf0c8c56 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeQueue.cpp @@ -9,6 +9,7 @@ #include #include #include +#include namespace DB @@ -19,7 +20,6 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; extern const int UNEXPECTED_NODE_IN_ZOOKEEPER; extern const int ABORTED; - extern const int READONLY; } @@ -145,7 +145,7 @@ bool ReplicatedMergeTreeQueue::load(zkutil::ZooKeeperPtr zookeeper) LOG_DEBUG(log, "Having {} queue entries to load, {} entries already loaded.", (to_remove_it - children.begin()), (children.end() - to_remove_it)); children.erase(to_remove_it, children.end()); - std::sort(children.begin(), children.end()); + ::sort(children.begin(), children.end()); zkutil::AsyncResponses futures; futures.reserve(children.size()); @@ -554,10 +554,17 @@ bool ReplicatedMergeTreeQueue::removeFailedQuorumPart(const MergeTreePartInfo & int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper, Coordination::WatchCallback watch_callback, PullLogsReason reason) { std::lock_guard lock(pull_logs_to_queue_mutex); - if (storage.is_readonly && reason == SYNC) + + if (reason != LOAD) { - throw Exception(ErrorCodes::READONLY, "Cannot SYNC REPLICA, because replica is readonly"); - /// TODO throw logical error for other reasons (except LOAD) + /// It's totally ok to load queue on readonly replica (that's what RestartingThread does on initialization). + /// It's ok if replica became readonly due to connection loss after we got current zookeeper (in this case zookeeper must be expired). + /// And it's ok if replica became readonly after shutdown. + /// In other cases it's likely that someone called pullLogsToQueue(...) when queue is not initialized yet by RestartingThread. + bool not_completely_initialized = storage.is_readonly && !zookeeper->expired() && !storage.shutdown_called; + if (not_completely_initialized) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Tried to pull logs to queue (reason: {}) on readonly replica {}, it's a bug", + reason, storage.getStorageID().getNameForLogs()); } if (pull_log_blocker.isCancelled()) @@ -600,7 +607,7 @@ int32_t ReplicatedMergeTreeQueue::pullLogsToQueue(zkutil::ZooKeeperPtr zookeeper if (!log_entries.empty()) { - std::sort(log_entries.begin(), log_entries.end()); + ::sort(log_entries.begin(), log_entries.end()); for (size_t entry_idx = 0, num_entries = log_entries.size(); entry_idx < num_entries;) { @@ -1059,12 +1066,12 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const LogEntry & en if (entry_for_same_part_it != future_parts.end()) { const LogEntry & another_entry = *entry_for_same_part_it->second; - const char * format_str = "Not executing log entry {} of type {} for part {} " - "because another log entry {} of type {} for the same part ({}) is being processed. This shouldn't happen often."; - LOG_INFO(log, format_str, entry.znode_name, entry.type, entry.new_part_name, - another_entry.znode_name, another_entry.type, another_entry.new_part_name); - out_reason = fmt::format(format_str, entry.znode_name, entry.type, entry.new_part_name, - another_entry.znode_name, another_entry.type, another_entry.new_part_name); + out_reason = fmt::format( + "Not executing log entry {} of type {} for part {} " + "because another log entry {} of type {} for the same part ({}) is being processed. This shouldn't happen often.", + entry.znode_name, entry.type, entry.new_part_name, + another_entry.znode_name, another_entry.type, another_entry.new_part_name); + LOG_INFO(log, fmt::runtime(out_reason)); return false; /** When the corresponding action is completed, then `isNotCoveredByFuturePart` next time, will succeed, @@ -1085,10 +1092,11 @@ bool ReplicatedMergeTreeQueue::isNotCoveredByFuturePartsImpl(const LogEntry & en if (future_part.contains(result_part)) { - const char * format_str = "Not executing log entry {} for part {} " - "because it is covered by part {} that is currently executing."; - LOG_TRACE(log, format_str, entry.znode_name, new_part_name, future_part_elem.first); - out_reason = fmt::format(format_str, entry.znode_name, new_part_name, future_part_elem.first); + out_reason = fmt::format( + "Not executing log entry {} for part {} " + "because it is covered by part {} that is currently executing.", + entry.znode_name, new_part_name, future_part_elem.first); + LOG_TRACE(log, fmt::runtime(out_reason)); return false; } } @@ -1170,11 +1178,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { if (future_parts.count(name)) { - const char * format_str = "Not executing log entry {} of type {} for part {} " - "because part {} is not ready yet (log entry for that part is being processed)."; - LOG_TRACE(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, name); - /// Copy-paste of above because we need structured logging (instead of already formatted message). - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, name); + out_postpone_reason = fmt::format( + "Not executing log entry {} of type {} for part {} " + "because part {} is not ready yet (log entry for that part is being processed).", + entry.znode_name, entry.typeToString(), entry.new_part_name, name); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); return false; } @@ -1190,12 +1198,30 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (merger_mutator.merges_blocker.isCancelled()) { - const char * format_str = "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now."; - LOG_DEBUG(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); + out_postpone_reason = fmt::format( + "Not executing log entry {} of type {} for part {} because merges and mutations are cancelled now.", + entry.znode_name, entry.typeToString(), entry.new_part_name); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } + const auto data_settings = data.getSettings(); + if (data_settings->allow_remote_fs_zero_copy_replication) + { + auto disks = storage.getDisks(); + bool only_s3_storage = true; + for (const auto & disk : disks) + if (disk->getType() != DB::DiskType::S3) + only_s3_storage = false; + + if (!disks.empty() && only_s3_storage && storage.checkZeroCopyLockExists(entry.new_part_name, disks[0])) + { + out_postpone_reason = "Not executing merge/mutation for the part " + entry.new_part_name + + ", waiting other replica to execute it and will fetch after."; + return false; + } + } + if (merge_strategy_picker.shouldMergeOnSingleReplica(entry)) { auto replica_to_execute_merge = merge_strategy_picker.pickReplicaToExecuteMerge(entry); @@ -1217,7 +1243,6 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( * Setting max_bytes_to_merge_at_max_space_in_pool still working for regular merges, * because the leader replica does not assign merges of greater size (except OPTIMIZE PARTITION and OPTIMIZE FINAL). */ - const auto data_settings = data.getSettings(); bool ignore_max_size = false; if (entry.type == LogEntry::MERGE_PARTS) { @@ -1227,24 +1252,20 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( { if (merger_mutator.ttl_merges_blocker.isCancelled()) { - const char * format_str = "Not executing log entry {} for part {} because merges with TTL are cancelled now."; - LOG_DEBUG(log, format_str, - entry.znode_name, entry.new_part_name); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.new_part_name); + out_postpone_reason = fmt::format( + "Not executing log entry {} for part {} because merges with TTL are cancelled now.", + entry.znode_name, entry.new_part_name); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } size_t total_merges_with_ttl = data.getTotalMergesWithTTLInMergeList(); if (total_merges_with_ttl >= data_settings->max_number_of_merges_with_ttl_in_pool) { - const char * format_str = "Not executing log entry {} for part {}" - " because {} merges with TTL already executing, maximum {}."; - LOG_DEBUG(log, format_str, - entry.znode_name, entry.new_part_name, total_merges_with_ttl, - data_settings->max_number_of_merges_with_ttl_in_pool); - - out_postpone_reason = fmt::format(format_str, + out_postpone_reason = fmt::format( + "Not executing log entry {} for part {} because {} merges with TTL already executing, maximum {}.", entry.znode_name, entry.new_part_name, total_merges_with_ttl, data_settings->max_number_of_merges_with_ttl_in_pool); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } } @@ -1257,7 +1278,7 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( entry.znode_name, entry.typeToString(), entry.new_part_name, ReadableSize(sum_parts_size_in_bytes), ReadableSize(max_source_parts_size)); - LOG_DEBUG(log, out_postpone_reason); + LOG_DEBUG(log, fmt::runtime(out_postpone_reason)); return false; } @@ -1270,9 +1291,10 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( if (!alter_sequence.canExecuteMetaAlter(entry.alter_version, state_lock)) { int head_alter = alter_sequence.getHeadAlterVersion(state_lock); - const char * format_str = "Cannot execute alter metadata {} with version {} because another alter {} must be executed before"; - LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version, head_alter); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version, head_alter); + out_postpone_reason = fmt::format( + "Cannot execute alter metadata {} with version {} because another alter {} must be executed before", + entry.znode_name, entry.alter_version, head_alter); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); return false; } } @@ -1285,15 +1307,17 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( int head_alter = alter_sequence.getHeadAlterVersion(state_lock); if (head_alter == entry.alter_version) { - const char * format_str = "Cannot execute alter data {} with version {} because metadata still not altered"; - LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version); + out_postpone_reason = fmt::format( + "Cannot execute alter data {} with version {} because metadata still not altered", + entry.znode_name, entry.alter_version); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); } else { - const char * format_str = "Cannot execute alter data {} with version {} because another alter {} must be executed before"; - LOG_TRACE(log, format_str, entry.znode_name, entry.alter_version, head_alter); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.alter_version, head_alter); + out_postpone_reason = fmt::format( + "Cannot execute alter data {} with version {} because another alter {} must be executed before", + entry.znode_name, entry.alter_version, head_alter); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); } return false; @@ -1308,11 +1332,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( /// See also removePartProducingOpsInRange(...) and ReplicatedMergeTreeQueue::CurrentlyExecuting. if (currently_executing_drop_or_replace_range) { - - const char * format_str = "Not executing log entry {} of type {} for part {} " - "because another DROP_RANGE or REPLACE_RANGE entry are currently executing."; - LOG_TRACE(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name); + out_postpone_reason = fmt::format( + "Not executing log entry {} of type {} for part {} " + "because another DROP_RANGE or REPLACE_RANGE entry are currently executing.", + entry.znode_name, entry.typeToString(), entry.new_part_name); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); return false; } @@ -1337,10 +1361,11 @@ bool ReplicatedMergeTreeQueue::shouldExecuteLogEntry( auto new_part_info = MergeTreePartInfo::fromPartName(new_part_name, format_version); if (!new_part_info.isDisjoint(drop_part_info)) { - const char * format_str = "Not executing log entry {} of type {} for part {} " - "because it probably depends on {} (REPLACE_RANGE)."; - LOG_TRACE(log, format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); - out_postpone_reason = fmt::format(format_str, entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); + out_postpone_reason = fmt::format( + "Not executing log entry {} of type {} for part {} " + "because it probably depends on {} (REPLACE_RANGE).", + entry.znode_name, entry.typeToString(), entry.new_part_name, replace_entry->znode_name); + LOG_TRACE(log, fmt::runtime(out_postpone_reason)); return false; } } @@ -1615,7 +1640,11 @@ MutationCommands ReplicatedMergeTreeQueue::getMutationCommands( auto end = in_partition->second.lower_bound(desired_mutation_version); if (end == in_partition->second.end() || end->first != desired_mutation_version) - LOG_WARNING(log, "Mutation with version {} not found in partition ID {} (trying to mutate part {}", desired_mutation_version, part->info.partition_id, part->name + ")"); + LOG_WARNING(log, + "Mutation with version {} not found in partition ID {} (trying to mutate part {})", + desired_mutation_version, + part->info.partition_id, + part->name); else ++end; @@ -1645,6 +1674,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep { LOG_TRACE(log, "Marking mutation {} done because it is <= mutation_pointer ({})", znode, mutation_pointer); mutation.is_done = true; + mutation.latest_fail_reason.clear(); alter_sequence.finishDataAlter(mutation.entry->alter_version, lock); if (mutation.parts_to_do.size() != 0) { @@ -1689,6 +1719,7 @@ bool ReplicatedMergeTreeQueue::tryFinalizeMutations(zkutil::ZooKeeperPtr zookeep { LOG_TRACE(log, "Mutation {} is done", entry->znode_name); it->second.is_done = true; + it->second.latest_fail_reason.clear(); if (entry->isAlterMutation()) { LOG_TRACE(log, "Finishing data alter with version {} for entry {}", entry->alter_version, entry->znode_name); diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp index 7a5b82979bd..de34929b43e 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.cpp @@ -58,106 +58,108 @@ void ReplicatedMergeTreeRestartingThread::run() if (need_stop) return; - bool reschedule_now = false; + size_t reschedule_period_ms = check_period_ms; + try { - if (first_time || readonly_mode_was_set || storage.getZooKeeper()->expired()) - { - startup_completed = false; - - if (first_time) - { - LOG_DEBUG(log, "Activating replica."); - } - else - { - if (storage.getZooKeeper()->expired()) - { - LOG_WARNING(log, "ZooKeeper session has expired. Switching to a new session."); - setReadonly(); - } - else if (readonly_mode_was_set) - { - LOG_WARNING(log, "Table was in readonly mode. Will try to activate it."); - } - partialShutdown(); - } - - if (!startup_completed) - { - try - { - storage.setZooKeeper(); - } - catch (const Coordination::Exception &) - { - /// The exception when you try to zookeeper_init usually happens if DNS does not work. We will try to do it again. - tryLogCurrentException(log, __PRETTY_FUNCTION__); - - /// Here we're almost sure the table is already readonly, but it doesn't hurt to enforce it. - setReadonly(); - if (first_time) - storage.startup_event.set(); - task->scheduleAfter(retry_period_ms); - return; - } - - if (!need_stop && !tryStartup()) - { - /// We couldn't startup replication. Table must be readonly. - /// Otherwise it can have partially initialized queue and other - /// strange parts of state. - setReadonly(); - - if (first_time) - storage.startup_event.set(); - - task->scheduleAfter(retry_period_ms); - return; - } - - if (first_time) - storage.startup_event.set(); - - startup_completed = true; - } - - if (need_stop) - return; - - bool old_val = true; - if (storage.is_readonly.compare_exchange_strong(old_val, false)) - { - readonly_mode_was_set = false; - CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica); - } - - first_time = false; - } + bool replica_is_active = runImpl(); + if (!replica_is_active) + reschedule_period_ms = retry_period_ms; } catch (const Exception & e) { /// We couldn't activate table let's set it into readonly mode - setReadonly(); partialShutdown(); - storage.startup_event.set(); tryLogCurrentException(log, __PRETTY_FUNCTION__); if (e.code() == ErrorCodes::REPLICA_STATUS_CHANGED) - reschedule_now = true; + reschedule_period_ms = 0; } catch (...) { - setReadonly(); partialShutdown(); - storage.startup_event.set(); tryLogCurrentException(log, __PRETTY_FUNCTION__); } - if (reschedule_now) - task->schedule(); + if (first_time) + { + if (storage.is_readonly) + { + /// We failed to start replication, table is still readonly, so we should increment the metric. See also setNotReadonly(). + CurrentMetrics::add(CurrentMetrics::ReadonlyReplica); + } + /// It does not matter if replication is actually started or not, just notify after the first attempt. + storage.startup_event.set(); + first_time = false; + } + + if (need_stop) + return; + + if (reschedule_period_ms) + task->scheduleAfter(reschedule_period_ms); else - task->scheduleAfter(check_period_ms); + task->schedule(); +} + +bool ReplicatedMergeTreeRestartingThread::runImpl() + +{ + if (!storage.is_readonly && !storage.getZooKeeper()->expired()) + return true; + + if (first_time) + { + LOG_DEBUG(log, "Activating replica."); + assert(storage.is_readonly); + } + else if (storage.is_readonly) + { + LOG_WARNING(log, "Table was in readonly mode. Will try to activate it."); + } + else if (storage.getZooKeeper()->expired()) + { + LOG_WARNING(log, "ZooKeeper session has expired. Switching to a new session."); + partialShutdown(); + } + else + { + __builtin_unreachable(); + } + + try + { + storage.setZooKeeper(); + } + catch (const Coordination::Exception &) + { + /// The exception when you try to zookeeper_init usually happens if DNS does not work. We will try to do it again. + tryLogCurrentException(log, __PRETTY_FUNCTION__); + assert(storage.is_readonly); + return false; + } + + if (need_stop) + return false; + + if (!tryStartup()) + { + assert(storage.is_readonly); + return false; + } + + setNotReadonly(); + + /// Start queue processing + storage.background_operations_assignee.start(); + storage.queue_updating_task->activateAndSchedule(); + storage.mutations_updating_task->activateAndSchedule(); + storage.mutations_finalizing_task->activateAndSchedule(); + storage.merge_selecting_task->activateAndSchedule(); + storage.cleanup_thread.start(); + storage.part_check_thread.start(); + + return true; } @@ -202,16 +204,6 @@ bool ReplicatedMergeTreeRestartingThread::tryStartup() storage.partial_shutdown_called = false; storage.partial_shutdown_event.reset(); - - /// Start queue processing - storage.background_operations_assignee.start(); - - storage.queue_updating_task->activateAndSchedule(); - storage.mutations_updating_task->activateAndSchedule(); - storage.mutations_finalizing_task->activateAndSchedule(); - storage.cleanup_thread.start(); - storage.part_check_thread.start(); - return true; } catch (...) @@ -365,8 +357,9 @@ void ReplicatedMergeTreeRestartingThread::activateReplica() } -void ReplicatedMergeTreeRestartingThread::partialShutdown() +void ReplicatedMergeTreeRestartingThread::partialShutdown(bool part_of_full_shutdown) { + setReadonly(part_of_full_shutdown); ProfileEvents::increment(ProfileEvents::ReplicaPartialShutdown); storage.partial_shutdown_called = true; @@ -375,6 +368,7 @@ void ReplicatedMergeTreeRestartingThread::partialShutdown() LOG_TRACE(log, "Waiting for threads to finish"); + storage.merge_selecting_task->deactivate(); storage.queue_updating_task->deactivate(); storage.mutations_updating_task->deactivate(); storage.mutations_finalizing_task->deactivate(); @@ -401,25 +395,35 @@ void ReplicatedMergeTreeRestartingThread::shutdown() task->deactivate(); LOG_TRACE(log, "Restarting thread finished"); - /// For detach table query, we should reset the ReadonlyReplica metric. - if (readonly_mode_was_set) - { - CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica); - readonly_mode_was_set = false; - } - /// Stop other tasks. - partialShutdown(); + partialShutdown(/* part_of_full_shutdown */ true); } -void ReplicatedMergeTreeRestartingThread::setReadonly() +void ReplicatedMergeTreeRestartingThread::setReadonly(bool on_shutdown) { bool old_val = false; - if (storage.is_readonly.compare_exchange_strong(old_val, true)) - { - readonly_mode_was_set = true; + bool became_readonly = storage.is_readonly.compare_exchange_strong(old_val, true); + + /// Do not increment the metric if replica became readonly due to shutdown. + if (became_readonly && on_shutdown) + return; + + if (became_readonly) CurrentMetrics::add(CurrentMetrics::ReadonlyReplica); - } + + /// Replica was already readonly, but we should decrement the metric, because we are detaching/dropping table. + if (on_shutdown) + CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica); +} + +void ReplicatedMergeTreeRestartingThread::setNotReadonly() +{ + bool old_val = true; + /// is_readonly is true on startup, but ReadonlyReplica metric is not incremented, + /// because we don't want to change this metric if replication is started successfully. + /// So we should not decrement it when replica stopped being readonly on startup. + if (storage.is_readonly.compare_exchange_strong(old_val, false) && !first_time) + CurrentMetrics::sub(CurrentMetrics::ReadonlyReplica); } } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h index bdade4e0bb2..e62cff4baf6 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeRestartingThread.h @@ -36,19 +36,18 @@ private: Poco::Logger * log; std::atomic need_stop {false}; - // We need it besides `storage.is_readonly`, because `shutdown()` may be called many times, that way `storage.is_readonly` will not change. - bool readonly_mode_was_set = false; - /// The random data we wrote into `/replicas/me/is_active`. String active_node_identifier; BackgroundSchedulePool::TaskHolder task; Int64 check_period_ms; /// The frequency of checking expiration of session in ZK. bool first_time = true; /// Activate replica for the first time. - bool startup_completed = false; void run(); + /// Restarts table if needed, returns false if it failed to restart replica. + bool runImpl(); + /// Start or stop background threads. Used for partial reinitialization when re-creating a session in ZooKeeper. bool tryStartup(); /// Returns false if ZooKeeper is not available. @@ -61,10 +60,13 @@ private: /// If there is an unreachable quorum, and we have a part, then add this replica to the quorum. void updateQuorumIfWeHavePart(); - void partialShutdown(); + void partialShutdown(bool part_of_full_shutdown = false); /// Set readonly mode for table - void setReadonly(); + void setReadonly(bool on_shutdown = false); + + /// Disable readonly mode for table + void setNotReadonly(); }; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp index d2bf6ba308b..86d36aab50c 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.cpp @@ -32,6 +32,17 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +struct ReplicatedMergeTreeSink::DelayedChunk +{ + struct Partition + { + MergeTreeDataWriter::TemporaryPart temp_part; + UInt64 elapsed_ns; + String block_id; + }; + + std::vector partitions; +}; ReplicatedMergeTreeSink::ReplicatedMergeTreeSink( StorageReplicatedMergeTree & storage_, @@ -60,6 +71,8 @@ ReplicatedMergeTreeSink::ReplicatedMergeTreeSink( quorum = 0; } +ReplicatedMergeTreeSink::~ReplicatedMergeTreeSink() = default; + /// Allow to verify that the session in ZooKeeper is still alive. static void assertSessionIsNotExpired(zkutil::ZooKeeperPtr & zookeeper) @@ -126,8 +139,6 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) { auto block = getHeader().cloneWithColumns(chunk.detachColumns()); - last_block_is_duplicate = false; - auto zookeeper = storage.getZooKeeper(); assertSessionIsNotExpired(zookeeper); @@ -140,6 +151,8 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) checkQuorumPrecondition(zookeeper); auto part_blocks = storage.writer.splitBlockIntoParts(block, max_parts_per_block, metadata_snapshot, context); + std::vector partitions; + String block_dedup_token; for (auto & current_block : part_blocks) { @@ -147,11 +160,11 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) /// Write part to the filesystem under temporary name. Calculate a checksum. - MergeTreeData::MutableDataPartPtr part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); + auto temp_part = storage.writer.writeTempPart(current_block, metadata_snapshot, context); /// If optimize_on_insert setting is true, current_block could become empty after merge /// and we didn't create part. - if (!part) + if (!temp_part.part) continue; String block_id; @@ -160,8 +173,16 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) { /// We add the hash from the data and partition identifier to deduplication ID. /// That is, do not insert the same data to the same partition twice. - block_id = part->getZeroLevelPartBlockID(); + const String & dedup_token = context->getSettingsRef().insert_deduplication_token; + if (!dedup_token.empty()) + { + /// multiple blocks can be inserted within the same insert query + /// an ordinal number is added to dedup token to generate a distinctive block id for each block + block_dedup_token = fmt::format("{}_{}", dedup_token, chunk_dedup_seqnum); + ++chunk_dedup_seqnum; + } + block_id = temp_part.part->getZeroLevelPartBlockID(block_dedup_token); LOG_DEBUG(log, "Wrote block with ID '{}', {} rows", block_id, current_block.block.rows()); } else @@ -169,27 +190,63 @@ void ReplicatedMergeTreeSink::consume(Chunk chunk) LOG_DEBUG(log, "Wrote block with {} rows", current_block.block.rows()); } + UInt64 elapsed_ns = watch.elapsed(); + + partitions.emplace_back(ReplicatedMergeTreeSink::DelayedChunk::Partition{ + .temp_part = std::move(temp_part), + .elapsed_ns = elapsed_ns, + .block_id = std::move(block_id) + }); + } + + finishDelayedChunk(zookeeper); + delayed_chunk = std::make_unique(); + delayed_chunk->partitions = std::move(partitions); + + /// If deduplicated data should not be inserted into MV, we need to set proper + /// value for `last_block_is_duplicate`, which is possible only after the part is committed. + /// Othervide we can delay commit. + /// TODO: we can also delay commit if there is no MVs. + if (!context->getSettingsRef().deduplicate_blocks_in_dependent_materialized_views) + finishDelayedChunk(zookeeper); +} + +void ReplicatedMergeTreeSink::finishDelayedChunk(zkutil::ZooKeeperPtr & zookeeper) +{ + if (!delayed_chunk) + return; + + last_block_is_duplicate = false; + + for (auto & partition : delayed_chunk->partitions) + { + partition.temp_part.finalize(); + + auto & part = partition.temp_part.part; + try { - commitPart(zookeeper, part, block_id); + commitPart(zookeeper, part, partition.block_id); + + last_block_is_duplicate = last_block_is_duplicate || part->is_duplicate; /// Set a special error code if the block is duplicate - int error = (deduplicate && last_block_is_duplicate) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; - PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus(error)); + int error = (deduplicate && part->is_duplicate) ? ErrorCodes::INSERT_WAS_DEDUPLICATED : 0; + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus(error)); } catch (...) { - PartLog::addNewPart(storage.getContext(), part, watch.elapsed(), ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); + PartLog::addNewPart(storage.getContext(), part, partition.elapsed_ns, ExecutionStatus::fromCurrentException(__PRETTY_FUNCTION__)); throw; } } + + delayed_chunk.reset(); } void ReplicatedMergeTreeSink::writeExistingPart(MergeTreeData::MutableDataPartPtr & part) { - last_block_is_duplicate = false; - /// NOTE: No delay in this case. That's Ok. auto zookeeper = storage.getZooKeeper(); @@ -347,7 +404,6 @@ void ReplicatedMergeTreeSink::commitPart( if (storage.getActiveContainingPart(existing_part_name)) { part->is_duplicate = true; - last_block_is_duplicate = true; ProfileEvents::increment(ProfileEvents::DuplicatedInsertedBlocks); if (quorum) { @@ -522,6 +578,12 @@ void ReplicatedMergeTreeSink::onStart() storage.delayInsertOrThrowIfNeeded(&storage.partial_shutdown_event); } +void ReplicatedMergeTreeSink::onFinish() +{ + auto zookeeper = storage.getZooKeeper(); + assertSessionIsNotExpired(zookeeper); + finishDelayedChunk(zookeeper); +} void ReplicatedMergeTreeSink::waitForQuorum( zkutil::ZooKeeperPtr & zookeeper, diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h index 7df82fd397e..41953e034fa 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeSink.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeSink.h @@ -35,8 +35,11 @@ public: // needed to set the special LogEntryType::ATTACH_PART bool is_attach_ = false); + ~ReplicatedMergeTreeSink() override; + void onStart() override; void consume(Chunk chunk) override; + void onFinish() override; String getName() const override { return "ReplicatedMergeTreeSink"; } @@ -82,13 +85,20 @@ private: bool is_attach = false; bool quorum_parallel = false; - bool deduplicate = true; + const bool deduplicate = true; bool last_block_is_duplicate = false; using Logger = Poco::Logger; Poco::Logger * log; ContextPtr context; + UInt64 chunk_dedup_seqnum = 0; /// input chunk ordinal number in case of dedup token + + /// We can delay processing for previous chunk and start writing a new one. + struct DelayedChunk; + std::unique_ptr delayed_chunk; + + void finishDelayedChunk(zkutil::ZooKeeperPtr & zookeeper); }; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp index 0637a6bb027..7dee7b8d0f8 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.cpp @@ -168,7 +168,7 @@ ReplicatedMergeTreeTableMetadata ReplicatedMergeTreeTableMetadata::parse(const S } -void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const ReplicatedMergeTreeTableMetadata & from_zk) const +void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const { if (data_format_version < MERGE_TREE_DATA_MIN_FORMAT_VERSION_WITH_CUSTOM_PARTITIONING) { @@ -203,9 +203,12 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat /// NOTE: You can make a less strict check of match expressions so that tables do not break from small changes /// in formatAST code. - if (primary_key != from_zk.primary_key) + String parsed_zk_primary_key = formattedAST(KeyDescription::parse(from_zk.primary_key, columns, context).expression_list_ast); + if (primary_key != parsed_zk_primary_key) throw Exception("Existing table metadata in ZooKeeper differs in primary key." - " Stored in ZooKeeper: " + from_zk.primary_key + ", local: " + primary_key, + " Stored in ZooKeeper: " + from_zk.primary_key + + ", parsed from ZooKeeper: " + parsed_zk_primary_key + + ", local: " + primary_key, ErrorCodes::METADATA_MISMATCH); if (data_format_version != from_zk.data_format_version) @@ -214,39 +217,53 @@ void ReplicatedMergeTreeTableMetadata::checkImmutableFieldsEquals(const Replicat ", local: " + DB::toString(data_format_version.toUnderType()), ErrorCodes::METADATA_MISMATCH); - if (partition_key != from_zk.partition_key) + String parsed_zk_partition_key = formattedAST(KeyDescription::parse(from_zk.partition_key, columns, context).expression_list_ast); + if (partition_key != parsed_zk_partition_key) throw Exception( "Existing table metadata in ZooKeeper differs in partition key expression." - " Stored in ZooKeeper: " + from_zk.partition_key + ", local: " + partition_key, + " Stored in ZooKeeper: " + from_zk.partition_key + + ", parsed from ZooKeeper: " + parsed_zk_partition_key + + ", local: " + partition_key, ErrorCodes::METADATA_MISMATCH); - } void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const { - checkImmutableFieldsEquals(from_zk); + checkImmutableFieldsEquals(from_zk, columns, context); - if (sampling_expression != from_zk.sampling_expression) - throw Exception("Existing table metadata in ZooKeeper differs in sample expression." - " Stored in ZooKeeper: " + from_zk.sampling_expression + ", local: " + sampling_expression, - ErrorCodes::METADATA_MISMATCH); - - if (sorting_key != from_zk.sorting_key) + String parsed_zk_sampling_expression = formattedAST(KeyDescription::parse(from_zk.sampling_expression, columns, context).definition_ast); + if (sampling_expression != parsed_zk_sampling_expression) { throw Exception( - "Existing table metadata in ZooKeeper differs in sorting key expression." - " Stored in ZooKeeper: " + from_zk.sorting_key + ", local: " + sorting_key, + "Existing table metadata in ZooKeeper differs in sample expression." + " Stored in ZooKeeper: " + from_zk.sampling_expression + + ", parsed from ZooKeeper: " + parsed_zk_sampling_expression + + ", local: " + sampling_expression, ErrorCodes::METADATA_MISMATCH); } - if (ttl_table != from_zk.ttl_table) + String parsed_zk_sorting_key = formattedAST(extractKeyExpressionList(KeyDescription::parse(from_zk.sorting_key, columns, context).definition_ast)); + if (sorting_key != parsed_zk_sorting_key) { throw Exception( - "Existing table metadata in ZooKeeper differs in TTL." - " Stored in ZooKeeper: " + from_zk.ttl_table + - ", local: " + ttl_table, - ErrorCodes::METADATA_MISMATCH); + "Existing table metadata in ZooKeeper differs in sorting key expression." + " Stored in ZooKeeper: " + from_zk.sorting_key + + ", parsed from ZooKeeper: " + parsed_zk_sorting_key + + ", local: " + sorting_key, + ErrorCodes::METADATA_MISMATCH); + } + + auto parsed_primary_key = KeyDescription::parse(primary_key, columns, context); + String parsed_zk_ttl_table = formattedAST(TTLTableDescription::parse(from_zk.ttl_table, columns, context, parsed_primary_key).definition_ast); + if (ttl_table != parsed_zk_ttl_table) + { + throw Exception( + "Existing table metadata in ZooKeeper differs in TTL." + " Stored in ZooKeeper: " + from_zk.ttl_table + + ", parsed from ZooKeeper: " + parsed_zk_ttl_table + + ", local: " + ttl_table, + ErrorCodes::METADATA_MISMATCH); } String parsed_zk_skip_indices = IndicesDescription::parse(from_zk.skip_indices, columns, context).toString(); @@ -290,10 +307,10 @@ void ReplicatedMergeTreeTableMetadata::checkEquals(const ReplicatedMergeTreeTabl } ReplicatedMergeTreeTableMetadata::Diff -ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk) const +ReplicatedMergeTreeTableMetadata::checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const { - checkImmutableFieldsEquals(from_zk); + checkImmutableFieldsEquals(from_zk, columns, context); Diff diff; diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h index 2f9a9d58834..6d510d20304 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h +++ b/src/Storages/MergeTree/ReplicatedMergeTreeTableMetadata.h @@ -70,11 +70,11 @@ struct ReplicatedMergeTreeTableMetadata void checkEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const; - Diff checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk) const; + Diff checkAndFindDiff(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const; private: - void checkImmutableFieldsEquals(const ReplicatedMergeTreeTableMetadata & from_zk) const; + void checkImmutableFieldsEquals(const ReplicatedMergeTreeTableMetadata & from_zk, const ColumnsDescription & columns, ContextPtr context) const; bool index_granularity_bytes_found_in_zk = false; }; diff --git a/src/Storages/MergeTree/localBackup.cpp b/src/Storages/MergeTree/localBackup.cpp index 236a0c5b5e4..1a04aa4b678 100644 --- a/src/Storages/MergeTree/localBackup.cpp +++ b/src/Storages/MergeTree/localBackup.cpp @@ -42,15 +42,31 @@ static void localBackupImpl(const DiskPtr & disk, const String & source_path, co } } +namespace +{ class CleanupOnFail { public: - explicit CleanupOnFail(std::function && cleaner_) : cleaner(cleaner_), is_success(false) {} + explicit CleanupOnFail(std::function && cleaner_) + : cleaner(cleaner_) + {} ~CleanupOnFail() { if (!is_success) - cleaner(); + { + /// We are trying to handle race condition here. So if we was not + /// able to backup directory try to remove garbage, but it's ok if + /// it doesn't exist. + try + { + cleaner(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } } void success() @@ -60,8 +76,9 @@ public: private: std::function cleaner; - bool is_success; + bool is_success{false}; }; +} void localBackup(const DiskPtr & disk, const String & source_path, const String & destination_path, std::optional max_level) { @@ -73,11 +90,11 @@ void localBackup(const DiskPtr & disk, const String & source_path, const String size_t try_no = 0; const size_t max_tries = 10; - CleanupOnFail cleanup([&](){disk->removeRecursive(destination_path);}); + CleanupOnFail cleanup([disk, destination_path]() { disk->removeRecursive(destination_path); }); /** Files in the directory can be permanently added and deleted. * If some file is deleted during an attempt to make a backup, then try again, - * because it's important to take into account any new files that might appear. + * because it's important to take into account any new files that might appear. */ while (true) { diff --git a/src/Storages/PartitionedSink.cpp b/src/Storages/PartitionedSink.cpp index 5e8f2a9e132..a42ea4ceff6 100644 --- a/src/Storages/PartitionedSink.cpp +++ b/src/Storages/PartitionedSink.cpp @@ -1,5 +1,7 @@ #include "PartitionedSink.h" +#include + #include #include @@ -40,19 +42,18 @@ PartitionedSink::PartitionedSink( } -SinkPtr PartitionedSink::getSinkForPartition(const String & partition_id) +SinkPtr PartitionedSink::getSinkForPartitionKey(StringRef partition_key) { - auto it = sinks.find(partition_id); - if (it == sinks.end()) + auto it = partition_id_to_sink.find(partition_key); + if (it == partition_id_to_sink.end()) { - auto sink = createSinkForPartition(partition_id); - std::tie(it, std::ignore) = sinks.emplace(partition_id, sink); + auto sink = createSinkForPartition(partition_key.toString()); + std::tie(it, std::ignore) = partition_id_to_sink.emplace(partition_key, sink); } return it->second; } - void PartitionedSink::consume(Chunk chunk) { const auto & columns = chunk.getColumns(); @@ -61,45 +62,59 @@ void PartitionedSink::consume(Chunk chunk) block_with_partition_by_expr.setColumns(columns); partition_by_expr->execute(block_with_partition_by_expr); - const auto * column = block_with_partition_by_expr.getByName(partition_by_column_name).column.get(); + const auto * partition_by_result_column = block_with_partition_by_expr.getByName(partition_by_column_name).column.get(); - std::unordered_map sub_chunks_indices; - IColumn::Selector selector; - for (size_t row = 0; row < chunk.getNumRows(); ++row) + size_t chunk_rows = chunk.getNumRows(); + chunk_row_index_to_partition_index.resize(chunk_rows); + + partition_id_to_chunk_index.clear(); + + for (size_t row = 0; row < chunk_rows; ++row) { - auto value = column->getDataAt(row); - auto [it, inserted] = sub_chunks_indices.emplace(value, sub_chunks_indices.size()); - selector.push_back(it->second); + auto partition_key = partition_by_result_column->getDataAt(row); + auto [it, inserted] = partition_id_to_chunk_index.insert(makePairNoInit(partition_key, partition_id_to_chunk_index.size())); + if (inserted) + it->value.first = copyStringInArena(partition_keys_arena, partition_key); + + chunk_row_index_to_partition_index[row] = it->getMapped(); } - Chunks sub_chunks; - sub_chunks.reserve(sub_chunks_indices.size()); - for (size_t column_index = 0; column_index < columns.size(); ++column_index) + size_t columns_size = columns.size(); + size_t partitions_size = partition_id_to_chunk_index.size(); + + Chunks partition_index_to_chunk; + partition_index_to_chunk.reserve(partitions_size); + + for (size_t column_index = 0; column_index < columns_size; ++column_index) { - MutableColumns column_sub_chunks = columns[column_index]->scatter(sub_chunks_indices.size(), selector); - if (column_index == 0) /// Set sizes for sub-chunks. + MutableColumns partition_index_to_column_split = columns[column_index]->scatter(partitions_size, chunk_row_index_to_partition_index); + + /// Add chunks into partition_index_to_chunk with sizes of result columns + if (column_index == 0) { - for (const auto & column_sub_chunk : column_sub_chunks) + for (const auto & partition_column : partition_index_to_column_split) { - sub_chunks.emplace_back(Columns(), column_sub_chunk->size()); + partition_index_to_chunk.emplace_back(Columns(), partition_column->size()); } } - for (size_t sub_chunk_index = 0; sub_chunk_index < column_sub_chunks.size(); ++sub_chunk_index) + + for (size_t partition_index = 0; partition_index < partitions_size; ++partition_index) { - sub_chunks[sub_chunk_index].addColumn(std::move(column_sub_chunks[sub_chunk_index])); + partition_index_to_chunk[partition_index].addColumn(std::move(partition_index_to_column_split[partition_index])); } } - for (const auto & [partition_id, sub_chunk_index] : sub_chunks_indices) + for (const auto & [partition_key, partition_index] : partition_id_to_chunk_index) { - getSinkForPartition(partition_id)->consume(std::move(sub_chunks[sub_chunk_index])); + auto sink = getSinkForPartitionKey(partition_key); + sink->consume(std::move(partition_index_to_chunk[partition_index])); } } void PartitionedSink::onFinish() { - for (auto & [partition_id, sink] : sinks) + for (auto & [_, sink] : partition_id_to_sink) { sink->onFinish(); } diff --git a/src/Storages/PartitionedSink.h b/src/Storages/PartitionedSink.h index bc59a603fac..7ed29f1b197 100644 --- a/src/Storages/PartitionedSink.h +++ b/src/Storages/PartitionedSink.h @@ -1,5 +1,8 @@ #pragma once +#include +#include +#include #include #include #include @@ -34,9 +37,13 @@ private: ExpressionActionsPtr partition_by_expr; String partition_by_column_name; - std::unordered_map sinks; + absl::flat_hash_map partition_id_to_sink; + HashMapWithSavedHash partition_id_to_chunk_index; + IColumn::Selector chunk_row_index_to_partition_index; + Arena partition_keys_arena; + + SinkPtr getSinkForPartitionKey(StringRef partition_key); - SinkPtr getSinkForPartition(const String & partition_id); }; } diff --git a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp index f02653d9167..db040584536 100644 --- a/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp +++ b/src/Storages/PostgreSQL/MaterializedPostgreSQLConsumer.cpp @@ -450,7 +450,8 @@ void MaterializedPostgreSQLConsumer::processReplicationMessage(const char * repl if (replica_identity != 'd' && replica_identity != 'i') { LOG_WARNING(log, - "Table has replica identity {} - not supported. A table must have a primary key or a replica identity index"); + "Table has replica identity {} - not supported. A table must have a primary key or a replica identity index", + replica_identity); markTableAsSkipped(relation_id, table_name); return; } diff --git a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp index e7d72de2056..876ba9b1698 100644 --- a/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp +++ b/src/Storages/PostgreSQL/PostgreSQLReplicationHandler.cpp @@ -1,5 +1,7 @@ #include "PostgreSQLReplicationHandler.h" +#include + #include #include #include @@ -698,7 +700,7 @@ std::set PostgreSQLReplicationHandler::fetchRequiredTables() } NameSet diff; - std::sort(expected_tables.begin(), expected_tables.end()); + ::sort(expected_tables.begin(), expected_tables.end()); std::set_symmetric_difference(expected_tables.begin(), expected_tables.end(), result_tables.begin(), result_tables.end(), std::inserter(diff, diff.begin())); diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index fe81b322bdb..c72dec824f0 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -277,10 +277,16 @@ Pipe StorageMaterializedPostgreSQL::read( size_t max_block_size, unsigned num_streams) { - auto materialized_table_lock = lockForShare(String(), context_->getSettingsRef().lock_acquire_timeout); auto nested_table = getNested(); - return readFinalFromNestedStorage(nested_table, column_names, metadata_snapshot, + + auto pipe = readFinalFromNestedStorage(nested_table, column_names, metadata_snapshot, query_info, context_, processed_stage, max_block_size, num_streams); + + auto lock = lockForShare(context_->getCurrentQueryId(), context_->getSettingsRef().lock_acquire_timeout); + pipe.addTableLock(lock); + pipe.addStorageHolder(shared_from_this()); + + return pipe; } diff --git a/src/Storages/ProjectionsDescription.cpp b/src/Storages/ProjectionsDescription.cpp index e13895e60f1..7c340cda739 100644 --- a/src/Storages/ProjectionsDescription.cpp +++ b/src/Storages/ProjectionsDescription.cpp @@ -107,7 +107,9 @@ ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const auto external_storage_holder = std::make_shared(query_context, columns, ConstraintsDescription{}); StoragePtr storage = external_storage_holder->getTable(); InterpreterSelectQuery select( - result.query_ast, query_context, storage, {}, SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias()); + result.query_ast, query_context, storage, {}, + /// Here we ignore ast optimizations because otherwise aggregation keys may be removed from result header as constants. + SelectQueryOptions{QueryProcessingStage::WithMergeableState}.modify().ignoreAlias().ignoreASTOptimizationsAlias()); result.required_columns = select.getRequiredColumns(); result.sample_block = select.getSampleBlock(); @@ -179,7 +181,7 @@ ProjectionDescription::getProjectionFromAST(const ASTPtr & definition_ast, const ProjectionDescription ProjectionDescription::getMinMaxCountProjection( const ColumnsDescription & columns, - const ASTPtr & partition_columns, + ASTPtr partition_columns, const Names & minmax_columns, const ASTs & primary_key_asts, ContextPtr query_context) @@ -203,7 +205,12 @@ ProjectionDescription ProjectionDescription::getMinMaxCountProjection( select_query->setExpression(ASTProjectionSelectQuery::Expression::SELECT, std::move(select_expression_list)); if (partition_columns && !partition_columns->children.empty()) + { + partition_columns = partition_columns->clone(); + for (const auto & partition_column : partition_columns->children) + KeyDescription::moduloToModuloLegacyRecursive(partition_column); select_query->setExpression(ASTProjectionSelectQuery::Expression::GROUP_BY, partition_columns->clone()); + } result.definition_ast = select_query; result.name = MINMAX_COUNT_PROJECTION_NAME; diff --git a/src/Storages/ProjectionsDescription.h b/src/Storages/ProjectionsDescription.h index 960e94e22f4..3e8d5e1a4f1 100644 --- a/src/Storages/ProjectionsDescription.h +++ b/src/Storages/ProjectionsDescription.h @@ -73,7 +73,7 @@ struct ProjectionDescription static ProjectionDescription getMinMaxCountProjection( const ColumnsDescription & columns, - const ASTPtr & partition_columns, + ASTPtr partition_columns, const Names & minmax_columns, const ASTs & primary_key_asts, ContextPtr query_context); diff --git a/src/Storages/ReadFinalForExternalReplicaStorage.cpp b/src/Storages/ReadFinalForExternalReplicaStorage.cpp index 36a40beca36..58b98aaa4c6 100644 --- a/src/Storages/ReadFinalForExternalReplicaStorage.cpp +++ b/src/Storages/ReadFinalForExternalReplicaStorage.cpp @@ -57,6 +57,7 @@ Pipe readFinalFromNestedStorage( Pipe pipe = nested_storage->read(require_columns_name, nested_metadata, query_info, context, processed_stage, max_block_size, num_streams); pipe.addTableLock(lock); + pipe.addStorageHolder(nested_storage); if (!expressions->children.empty() && !pipe.empty()) { diff --git a/src/Storages/ReadInOrderOptimizer.cpp b/src/Storages/ReadInOrderOptimizer.cpp index bae24f97b28..9e46595e0fc 100644 --- a/src/Storages/ReadInOrderOptimizer.cpp +++ b/src/Storages/ReadInOrderOptimizer.cpp @@ -6,6 +6,8 @@ #include #include #include +#include +#include namespace DB { @@ -15,13 +17,152 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +namespace +{ + +ASTPtr getFixedPoint(const ASTPtr & ast) +{ + const auto * func = ast->as(); + if (!func || func->name != "equals") + return nullptr; + + const auto & lhs = func->arguments->children[0]; + const auto & rhs = func->arguments->children[1]; + + if (lhs->as()) + return rhs; + + if (rhs->as()) + return lhs; + + return nullptr; +} + +size_t calculateFixedPrefixSize( + const ASTSelectQuery & query, const Names & sorting_key_columns) +{ + ASTPtr condition; + if (query.where() && query.prewhere()) + condition = makeASTFunction("and", query.where(), query.prewhere()); + else if (query.where()) + condition = query.where(); + else if (query.prewhere()) + condition = query.prewhere(); + + if (!condition) + return 0; + + /// Convert condition to CNF for more convenient analysis. + auto cnf = TreeCNFConverter::tryConvertToCNF(condition); + if (!cnf) + return 0; + + NameSet fixed_points; + + /// If we met expression like 'column = x', where 'x' is literal, + /// in clause of size 1 in CNF, then we can guarantee + /// that in all filtered rows 'column' will be equal to 'x'. + cnf->iterateGroups([&](const auto & group) + { + if (group.size() == 1 && !group.begin()->negative) + { + auto fixed_point = getFixedPoint(group.begin()->ast); + if (fixed_point) + fixed_points.insert(fixed_point->getColumnName()); + } + }); + + size_t prefix_size = 0; + for (const auto & column_name : sorting_key_columns) + { + if (!fixed_points.contains(column_name)) + break; + + ++prefix_size; + } + + return prefix_size; +} + +/// Optimize in case of exact match with order key element +/// or in some simple cases when order key element is wrapped into monotonic function. +/// Returns on of {-1, 0, 1} - direction of the match. 0 means - doesn't match. +int matchSortDescriptionAndKey( + const ExpressionActions::Actions & actions, + const SortColumnDescription & sort_column, + const String & sorting_key_column) +{ + /// If required order depend on collation, it cannot be matched with primary key order. + /// Because primary keys cannot have collations. + if (sort_column.collator) + return 0; + + int current_direction = sort_column.direction; + /// For the path: order by (sort_column, ...) + if (sort_column.column_name == sorting_key_column) + return current_direction; + + /// For the path: order by (function(sort_column), ...) + /// Allow only one simple monotonic functions with one argument + /// Why not allow multi monotonic functions? + bool found_function = false; + + for (const auto & action : actions) + { + if (action.node->type != ActionsDAG::ActionType::FUNCTION) + continue; + + if (found_function) + { + current_direction = 0; + break; + } + else + { + found_function = true; + } + + if (action.node->children.size() != 1 || action.node->children.at(0)->result_name != sorting_key_column) + { + current_direction = 0; + break; + } + + const auto & func = *action.node->function_base; + if (!func.hasInformationAboutMonotonicity()) + { + current_direction = 0; + break; + } + + auto monotonicity = func.getMonotonicityForRange(*func.getArgumentTypes().at(0), {}, {}); + if (!monotonicity.is_monotonic) + { + current_direction = 0; + break; + } + else if (!monotonicity.is_positive) + { + current_direction *= -1; + } + } + + if (!found_function) + current_direction = 0; + + return current_direction; +} + +} ReadInOrderOptimizer::ReadInOrderOptimizer( + const ASTSelectQuery & query_, const ManyExpressionActions & elements_actions_, const SortDescription & required_sort_description_, const TreeRewriterResultPtr & syntax_result) : elements_actions(elements_actions_) , required_sort_description(required_sort_description_) + , query(query_) { if (elements_actions.size() != required_sort_description.size()) throw Exception("Sizes of sort description and actions are mismatched", ErrorCodes::LOGICAL_ERROR); @@ -35,126 +176,88 @@ ReadInOrderOptimizer::ReadInOrderOptimizer( array_join_result_to_source = syntax_result->array_join_result_to_source; } -InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context, UInt64 limit) const +InputOrderInfoPtr ReadInOrderOptimizer::getInputOrderImpl( + const StorageMetadataPtr & metadata_snapshot, + const SortDescription & description, + const ManyExpressionActions & actions, + UInt64 limit) const { - Names sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); - if (!metadata_snapshot->hasSortingKey()) - return {}; + auto sorting_key_columns = metadata_snapshot->getSortingKeyColumns(); + int read_direction = description.at(0).direction; + + size_t fixed_prefix_size = calculateFixedPrefixSize(query, sorting_key_columns); + size_t descr_prefix_size = std::min(description.size(), sorting_key_columns.size() - fixed_prefix_size); SortDescription order_key_prefix_descr; - int read_direction = required_sort_description.at(0).direction; + order_key_prefix_descr.reserve(descr_prefix_size); - size_t prefix_size = std::min(required_sort_description.size(), sorting_key_columns.size()); - auto aliased_columns = metadata_snapshot->getColumns().getAliases(); - - for (size_t i = 0; i < prefix_size; ++i) + for (size_t i = 0; i < descr_prefix_size; ++i) { - if (forbidden_columns.count(required_sort_description[i].column_name)) + if (forbidden_columns.count(description[i].column_name)) break; - /// Optimize in case of exact match with order key element - /// or in some simple cases when order key element is wrapped into monotonic function. - auto apply_order_judge = [&] (const ExpressionActions::Actions & actions, const String & sort_column) - { - /// If required order depend on collation, it cannot be matched with primary key order. - /// Because primary keys cannot have collations. - if (required_sort_description[i].collator) - return false; + int current_direction = matchSortDescriptionAndKey( + actions[i]->getActions(), description[i], sorting_key_columns[i + fixed_prefix_size]); - int current_direction = required_sort_description[i].direction; - /// For the path: order by (sort_column, ...) - if (sort_column == sorting_key_columns[i] && current_direction == read_direction) - { - return true; - } - /// For the path: order by (function(sort_column), ...) - /// Allow only one simple monotonic functions with one argument - /// Why not allow multi monotonic functions? - else - { - bool found_function = false; - - for (const auto & action : actions) - { - if (action.node->type != ActionsDAG::ActionType::FUNCTION) - { - continue; - } - - if (found_function) - { - current_direction = 0; - break; - } - else - found_function = true; - - if (action.node->children.size() != 1 || action.node->children.at(0)->result_name != sorting_key_columns[i]) - { - current_direction = 0; - break; - } - - const auto & func = *action.node->function_base; - if (!func.hasInformationAboutMonotonicity()) - { - current_direction = 0; - break; - } - - auto monotonicity = func.getMonotonicityForRange(*func.getArgumentTypes().at(0), {}, {}); - if (!monotonicity.is_monotonic) - { - current_direction = 0; - break; - } - else if (!monotonicity.is_positive) - current_direction *= -1; - } - - if (!found_function) - current_direction = 0; - - if (!current_direction || (i > 0 && current_direction != read_direction)) - return false; - - if (i == 0) - read_direction = current_direction; - - return true; - } - }; - - const auto & actions = elements_actions[i]->getActions(); - bool ok; - /// check if it's alias column - /// currently we only support alias column without any function wrapper - /// ie: `order by aliased_column` can have this optimization, but `order by function(aliased_column)` can not. - /// This suits most cases. - if (context->getSettingsRef().optimize_respect_aliases && aliased_columns.contains(required_sort_description[i].column_name)) - { - auto column_expr = metadata_snapshot->getColumns().get(required_sort_description[i].column_name).default_desc.expression->clone(); - replaceAliasColumnsInQuery(column_expr, metadata_snapshot->getColumns(), array_join_result_to_source, context); - - auto syntax_analyzer_result = TreeRewriter(context).analyze(column_expr, metadata_snapshot->getColumns().getAll()); - const auto expression_analyzer = ExpressionAnalyzer(column_expr, syntax_analyzer_result, context).getActions(true); - const auto & alias_actions = expression_analyzer->getActions(); - - ok = apply_order_judge(alias_actions, column_expr->getColumnName()); - } - else - ok = apply_order_judge(actions, required_sort_description[i].column_name); - - if (ok) - order_key_prefix_descr.push_back(required_sort_description[i]); - else + if (!current_direction || (i > 0 && current_direction != read_direction)) break; + + if (i == 0) + read_direction = current_direction; + + order_key_prefix_descr.push_back(required_sort_description[i]); } if (order_key_prefix_descr.empty()) return {}; - return std::make_shared(std::move(order_key_prefix_descr), read_direction, limit); + SortDescription order_key_fixed_prefix_descr; + order_key_fixed_prefix_descr.reserve(fixed_prefix_size); + for (size_t i = 0; i < fixed_prefix_size; ++i) + order_key_fixed_prefix_descr.emplace_back(sorting_key_columns[i], read_direction); + + return std::make_shared( + std::move(order_key_fixed_prefix_descr), + std::move(order_key_prefix_descr), + read_direction, limit); +} + +InputOrderInfoPtr ReadInOrderOptimizer::getInputOrder( + const StorageMetadataPtr & metadata_snapshot, ContextPtr context, UInt64 limit) const +{ + if (!metadata_snapshot->hasSortingKey()) + return {}; + + auto aliased_columns = metadata_snapshot->getColumns().getAliases(); + + /// Replace alias column with proper expressions. + /// Currently we only support alias column without any function wrapper, + /// i.e.: `order by aliased_column` can have this optimization, but `order by function(aliased_column)` can not. + /// This suits most cases. + if (context->getSettingsRef().optimize_respect_aliases && !aliased_columns.empty()) + { + SortDescription aliases_sort_description = required_sort_description; + ManyExpressionActions aliases_actions = elements_actions; + + for (size_t i = 0; i < required_sort_description.size(); ++i) + { + if (!aliased_columns.contains(required_sort_description[i].column_name)) + continue; + + auto column_expr = metadata_snapshot->getColumns().get(required_sort_description[i].column_name).default_desc.expression->clone(); + replaceAliasColumnsInQuery(column_expr, metadata_snapshot->getColumns(), array_join_result_to_source, context); + + auto syntax_analyzer_result = TreeRewriter(context).analyze(column_expr, metadata_snapshot->getColumns().getAll()); + auto expression_analyzer = ExpressionAnalyzer(column_expr, syntax_analyzer_result, context); + + aliases_sort_description[i].column_name = column_expr->getColumnName(); + aliases_actions[i] = expression_analyzer.getActions(true); + } + + return getInputOrderImpl(metadata_snapshot, aliases_sort_description, aliases_actions, limit); + } + + return getInputOrderImpl(metadata_snapshot, required_sort_description, elements_actions, limit); } } diff --git a/src/Storages/ReadInOrderOptimizer.h b/src/Storages/ReadInOrderOptimizer.h index 2686d081855..fd8c9187ddb 100644 --- a/src/Storages/ReadInOrderOptimizer.h +++ b/src/Storages/ReadInOrderOptimizer.h @@ -18,6 +18,7 @@ class ReadInOrderOptimizer { public: ReadInOrderOptimizer( + const ASTSelectQuery & query, const ManyExpressionActions & elements_actions, const SortDescription & required_sort_description, const TreeRewriterResultPtr & syntax_result); @@ -25,10 +26,17 @@ public: InputOrderInfoPtr getInputOrder(const StorageMetadataPtr & metadata_snapshot, ContextPtr context, UInt64 limit = 0) const; private: + InputOrderInfoPtr getInputOrderImpl( + const StorageMetadataPtr & metadata_snapshot, + const SortDescription & description, + const ManyExpressionActions & actions, + UInt64 limit) const; + /// Actions for every element of order expression to analyze functions for monotonicity ManyExpressionActions elements_actions; NameSet forbidden_columns; NameToNameMap array_join_result_to_source; SortDescription required_sort_description; + const ASTSelectQuery & query; }; } diff --git a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp index 18bf0e2c19b..3dbb5b18de9 100644 --- a/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp +++ b/src/Storages/RocksDB/StorageEmbeddedRocksDB.cpp @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -457,7 +458,7 @@ Pipe StorageEmbeddedRocksDB::read( if (keys->empty()) return {}; - std::sort(keys->begin(), keys->end()); + ::sort(keys->begin(), keys->end()); keys->erase(std::unique(keys->begin(), keys->end()), keys->end()); Pipes pipes; diff --git a/src/Storages/SelectQueryInfo.h b/src/Storages/SelectQueryInfo.h index 5df50ab9a7c..f15f2dd2626 100644 --- a/src/Storages/SelectQueryInfo.h +++ b/src/Storages/SelectQueryInfo.h @@ -87,19 +87,22 @@ struct FilterDAGInfo struct InputOrderInfo { + SortDescription order_key_fixed_prefix_descr; SortDescription order_key_prefix_descr; int direction; UInt64 limit; - InputOrderInfo(const SortDescription & order_key_prefix_descr_, int direction_, UInt64 limit_) - : order_key_prefix_descr(order_key_prefix_descr_), direction(direction_), limit(limit_) {} - - bool operator ==(const InputOrderInfo & other) const + InputOrderInfo( + const SortDescription & order_key_fixed_prefix_descr_, + const SortDescription & order_key_prefix_descr_, + int direction_, UInt64 limit_) + : order_key_fixed_prefix_descr(order_key_fixed_prefix_descr_) + , order_key_prefix_descr(order_key_prefix_descr_) + , direction(direction_), limit(limit_) { - return order_key_prefix_descr == other.order_key_prefix_descr && direction == other.direction; } - bool operator !=(const InputOrderInfo & other) const { return !(*this == other); } + bool operator==(const InputOrderInfo &) const = default; }; class IMergeTreeDataPart; @@ -124,6 +127,7 @@ struct ProjectionCandidate ReadInOrderOptimizerPtr order_optimizer; InputOrderInfoPtr input_order_info; ManyExpressionActions group_by_elements_actions; + SortDescription group_by_elements_order_descr; std::shared_ptr subqueries_for_sets; MergeTreeDataSelectAnalysisResultPtr merge_tree_projection_select_result_ptr; MergeTreeDataSelectAnalysisResultPtr merge_tree_normal_select_result_ptr; diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index 29772a14752..f97c09471c3 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -199,6 +199,8 @@ QueryProcessingStage::Enum StorageBuffer::getQueryProcessingStage( if (destination.get() == this) throw Exception("Destination table is myself. Read will cause infinite loop.", ErrorCodes::INFINITE_LOOP); + /// TODO: Find a way to support projections for StorageBuffer + query_info.ignore_projections = true; return destination->getQueryProcessingStage(local_context, to_stage, destination->getInMemoryMetadataPtr(), query_info); } @@ -365,9 +367,10 @@ void StorageBuffer::read( */ if (processed_stage > QueryProcessingStage::FetchColumns) { + /// TODO: Find a way to support projections for StorageBuffer auto interpreter = InterpreterSelectQuery( query_info.query, local_context, std::move(pipe_from_buffers), - SelectQueryOptions(processed_stage)); + SelectQueryOptions(processed_stage).ignoreProjections()); interpreter.buildQueryPlan(buffers_plan); } else @@ -449,46 +452,34 @@ void StorageBuffer::read( static void appendBlock(const Block & from, Block & to) { - if (!to) - throw Exception("Cannot append to empty block", ErrorCodes::LOGICAL_ERROR); + size_t rows = from.rows(); + size_t old_rows = to.rows(); + size_t old_bytes = to.bytes(); - if (to.rows()) - assertBlocksHaveEqualStructure(from, to, "Buffer"); + if (!to) + to = from.cloneEmpty(); + + assertBlocksHaveEqualStructure(from, to, "Buffer"); from.checkNumberOfRows(); to.checkNumberOfRows(); - size_t rows = from.rows(); - size_t bytes = from.bytes(); - - size_t old_rows = to.rows(); - size_t old_bytes = to.bytes(); - MutableColumnPtr last_col; try { MemoryTrackerBlockerInThread temporarily_disable_memory_tracker; - if (to.rows() == 0) + for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) { - to = from; - CurrentMetrics::add(CurrentMetrics::StorageBufferRows, rows); - CurrentMetrics::add(CurrentMetrics::StorageBufferBytes, bytes); - } - else - { - for (size_t column_no = 0, columns = to.columns(); column_no < columns; ++column_no) - { - const IColumn & col_from = *from.getByPosition(column_no).column.get(); - last_col = IColumn::mutate(std::move(to.getByPosition(column_no).column)); + const IColumn & col_from = *from.getByPosition(column_no).column.get(); + last_col = IColumn::mutate(std::move(to.getByPosition(column_no).column)); - last_col->insertRangeFrom(col_from, 0, rows); + last_col->insertRangeFrom(col_from, 0, rows); - to.getByPosition(column_no).column = std::move(last_col); - } - CurrentMetrics::add(CurrentMetrics::StorageBufferRows, rows); - CurrentMetrics::add(CurrentMetrics::StorageBufferBytes, to.bytes() - old_bytes); + to.getByPosition(column_no).column = std::move(last_col); } + CurrentMetrics::add(CurrentMetrics::StorageBufferRows, rows); + CurrentMetrics::add(CurrentMetrics::StorageBufferBytes, to.bytes() - old_bytes); } catch (...) { @@ -626,14 +617,7 @@ private: /// Sort the columns in the block. This is necessary to make it easier to concatenate the blocks later. Block sorted_block = block.sortColumns(); - if (!buffer.data) - { - buffer.data = sorted_block.cloneEmpty(); - - storage.total_writes.rows += buffer.data.rows(); - storage.total_writes.bytes += buffer.data.allocatedBytes(); - } - else if (storage.checkThresholds(buffer, /* direct= */true, current_time, sorted_block.rows(), sorted_block.bytes())) + if (storage.checkThresholds(buffer, /* direct= */true, current_time, sorted_block.rows(), sorted_block.bytes())) { /** If, after inserting the buffer, the constraints are exceeded, then we will reset the buffer. * This also protects against unlimited consumption of RAM, since if it is impossible to write to the table, @@ -735,7 +719,7 @@ bool StorageBuffer::optimize( if (deduplicate) throw Exception("DEDUPLICATE cannot be specified when optimizing table of type Buffer", ErrorCodes::NOT_IMPLEMENTED); - flushAllBuffers(false, true); + flushAllBuffers(false); return true; } @@ -813,42 +797,32 @@ bool StorageBuffer::checkThresholdsImpl(bool direct, size_t rows, size_t bytes, } -void StorageBuffer::flushAllBuffers(bool check_thresholds, bool reset_blocks_structure) +void StorageBuffer::flushAllBuffers(bool check_thresholds) { for (auto & buf : buffers) - flushBuffer(buf, check_thresholds, false, reset_blocks_structure); + flushBuffer(buf, check_thresholds, false); } -void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool locked, bool reset_block_structure) +bool StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool locked) { Block block_to_write; time_t current_time = time(nullptr); - size_t rows = 0; - size_t bytes = 0; - time_t time_passed = 0; - std::optional> lock; if (!locked) lock.emplace(buffer.lockForReading()); - block_to_write = buffer.data.cloneEmpty(); - - rows = buffer.data.rows(); - bytes = buffer.data.bytes(); + time_t time_passed = 0; + size_t rows = buffer.data.rows(); + size_t bytes = buffer.data.bytes(); if (buffer.first_write_time) time_passed = current_time - buffer.first_write_time; if (check_thresholds) { if (!checkThresholdsImpl(/* direct= */false, rows, bytes, time_passed)) - return; - } - else - { - if (rows == 0) - return; + return false; } buffer.data.swap(block_to_write); @@ -869,7 +843,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc total_writes.bytes -= block_allocated_bytes_delta; LOG_DEBUG(log, "Flushing buffer with {} rows (discarded), {} bytes, age {} seconds {}.", rows, bytes, time_passed, (check_thresholds ? "(bg)" : "(direct)")); - return; + return true; } /** For simplicity, buffer is locked during write. @@ -883,8 +857,6 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc try { writeBlockToDestination(block_to_write, DatabaseCatalog::instance().tryGetTable(destination_id, getContext())); - if (reset_block_structure) - buffer.data.clear(); } catch (...) { @@ -909,6 +881,7 @@ void StorageBuffer::flushBuffer(Buffer & buffer, bool check_thresholds, bool loc UInt64 milliseconds = watch.elapsedMilliseconds(); LOG_DEBUG(log, "Flushing buffer with {} rows, {} bytes, age {} seconds, took {} ms {}.", rows, bytes, time_passed, milliseconds, (check_thresholds ? "(bg)" : "(direct)")); + return true; } diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 28f8de5fd88..f04619a1d21 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -153,11 +153,8 @@ private: Poco::Logger * log; - void flushAllBuffers(bool check_thresholds = true, bool reset_blocks_structure = false); - /// Reset the buffer. If check_thresholds is set - resets only if thresholds - /// are exceeded. If reset_block_structure is set - clears inner block - /// structure inside buffer (useful in OPTIMIZE and ALTER). - void flushBuffer(Buffer & buffer, bool check_thresholds, bool locked = false, bool reset_block_structure = false); + void flushAllBuffers(bool check_thresholds = true); + bool flushBuffer(Buffer & buffer, bool check_thresholds, bool locked = false); bool checkThresholds(const Buffer & buffer, bool direct, time_t current_time, size_t additional_rows = 0, size_t additional_bytes = 0) const; bool checkThresholdsImpl(bool direct, size_t rows, size_t bytes, time_t time_passed) const; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index bcb12cc86b0..da648aa4e5c 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -302,13 +302,13 @@ NamesAndTypesList StorageDistributed::getVirtuals() const /// NOTE This is weird. Most of these virtual columns are part of MergeTree /// tables info. But Distributed is general-purpose engine. return NamesAndTypesList{ - NameAndTypePair("_table", std::make_shared()), - NameAndTypePair("_part", std::make_shared()), - NameAndTypePair("_part_index", std::make_shared()), - NameAndTypePair("_part_uuid", std::make_shared()), - NameAndTypePair("_partition_id", std::make_shared()), - NameAndTypePair("_sample_factor", std::make_shared()), - NameAndTypePair("_shard_num", std::make_shared()), + NameAndTypePair("_table", std::make_shared()), + NameAndTypePair("_part", std::make_shared()), + NameAndTypePair("_part_index", std::make_shared()), + NameAndTypePair("_part_uuid", std::make_shared()), + NameAndTypePair("_partition_id", std::make_shared()), + NameAndTypePair("_sample_factor", std::make_shared()), + NameAndTypePair("_shard_num", std::make_shared()), /// deprecated }; } @@ -605,8 +605,8 @@ Pipe StorageDistributed::read( void StorageDistributed::read( QueryPlan & query_plan, - const Names & column_names, - const StorageMetadataPtr & metadata_snapshot, + const Names &, + const StorageMetadataPtr &, SelectQueryInfo & query_info, ContextPtr local_context, QueryProcessingStage::Enum processed_stage, @@ -635,10 +635,6 @@ void StorageDistributed::read( return; } - bool has_virtual_shard_num_column = std::find(column_names.begin(), column_names.end(), "_shard_num") != column_names.end(); - if (has_virtual_shard_num_column && !isVirtualColumn("_shard_num", metadata_snapshot)) - has_virtual_shard_num_column = false; - StorageID main_table = StorageID::createEmpty(); if (!remote_table_function_ptr) main_table = StorageID{remote_database, remote_table}; @@ -646,8 +642,7 @@ void StorageDistributed::read( ClusterProxy::SelectStreamFactory select_stream_factory = ClusterProxy::SelectStreamFactory( header, - processed_stage, - has_virtual_shard_num_column); + processed_stage); ClusterProxy::executeQuery( query_plan, header, processed_stage, diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index eae46220c86..c2f6fb1608d 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -66,6 +66,7 @@ StoragePtr StorageFactory::get( bool has_force_restore_data_flag) const { String name, comment; + ASTStorage * storage_def = query.storage; bool has_engine_args = false; @@ -107,7 +108,10 @@ StoragePtr StorageFactory::get( } else { - if (!storage_def) + if (!query.storage) + throw Exception("Incorrect CREATE query: storage required", ErrorCodes::INCORRECT_QUERY); + + if (!storage_def->engine) throw Exception("Incorrect CREATE query: ENGINE required", ErrorCodes::ENGINE_REQUIRED); const ASTFunction & engine_def = *storage_def->engine; diff --git a/src/Storages/StorageFile.cpp b/src/Storages/StorageFile.cpp index e6c5f25dd57..9a2ec0789cd 100644 --- a/src/Storages/StorageFile.cpp +++ b/src/Storages/StorageFile.cpp @@ -199,18 +199,27 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user fs_table_path = user_files_absolute_path / fs_table_path; Strings paths; + /// Do not use fs::canonical or fs::weakly_canonical. /// Otherwise it will not allow to work with symlinks in `user_files_path` directory. String path = fs::absolute(fs_table_path).lexically_normal(); /// Normalize path. - if (path.find_first_of("*?{") == std::string::npos) + + if (path.find(PartitionedSink::PARTITION_ID_WILDCARD) != std::string::npos) + { + paths.push_back(path); + } + else if (path.find_first_of("*?{") == std::string::npos) { std::error_code error; if (fs::exists(path)) total_bytes_to_read += fs::file_size(path, error); + paths.push_back(path); } else + { paths = listFilesWithRegexpMatching("/", path, total_bytes_to_read); + } for (const auto & cur_path : paths) checkCreationIsAllowed(context, user_files_absolute_path, cur_path); @@ -218,8 +227,36 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user return paths; } +ColumnsDescription StorageFile::getTableStructureFromFileDescriptor(ContextPtr context) +{ + /// If we want to read schema from file descriptor we should create + /// a read buffer from fd, create a checkpoint, read some data required + /// for schema inference, rollback to checkpoint and then use the created + /// peekable read buffer on the first read from storage. It's needed because + /// in case of file descriptor we have a stream of data and we cannot + /// start reading data from the beginning after reading some data for + /// schema inference. + auto read_buffer_creator = [&]() + { + /// We will use PeekableReadBuffer to create a checkpoint, so we need a place + /// where we can store the original read buffer. + read_buffer_from_fd = createReadBuffer("", true, getName(), table_fd, compression_method, context); + auto read_buf = std::make_unique(*read_buffer_from_fd); + read_buf->setCheckpoint(); + return read_buf; + }; -ColumnsDescription StorageFile::getTableStructureFromData( + auto columns = readSchemaFromFormat(format_name, format_settings, read_buffer_creator, context, peekable_read_buffer_from_fd); + if (peekable_read_buffer_from_fd) + { + /// If we have created read buffer in readSchemaFromFormat we should rollback to checkpoint. + assert_cast(peekable_read_buffer_from_fd.get())->rollbackToCheckpoint(); + has_peekable_read_buffer_from_fd = true; + } + return columns; +} + +ColumnsDescription StorageFile::getTableStructureFromFile( const String & format, const std::vector & paths, const String & compression_method, @@ -272,8 +309,6 @@ StorageFile::StorageFile(int table_fd_, CommonArguments args) throw Exception("Using file descriptor as source of storage isn't allowed for server daemons", ErrorCodes::DATABASE_ACCESS_DENIED); if (args.format_name == "Distributed") throw Exception("Distributed format is allowed only with explicit file path", ErrorCodes::INCORRECT_FILE_NAME); - if (args.columns.empty()) - throw Exception("Automatic schema inference is not allowed when using file descriptor as source of storage", ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE); is_db_table = false; use_table_fd = true; @@ -287,7 +322,11 @@ StorageFile::StorageFile(const std::string & table_path_, const std::string & us is_db_table = false; paths = getPathsList(table_path_, user_files_path, args.getContext(), total_bytes_to_read); is_path_with_globs = paths.size() > 1; - path_for_partitioned_write = table_path_; + if (!paths.empty()) + path_for_partitioned_write = paths.front(); + else + path_for_partitioned_write = table_path_; + setStorageMetadata(args); } @@ -323,9 +362,15 @@ void StorageFile::setStorageMetadata(CommonArguments args) if (args.format_name == "Distributed" || args.columns.empty()) { - auto columns = getTableStructureFromData(format_name, paths, compression_method, format_settings, args.getContext()); - if (!args.columns.empty() && args.columns != columns) - throw Exception("Table structure and file structure are different", ErrorCodes::INCOMPATIBLE_COLUMNS); + ColumnsDescription columns; + if (use_table_fd) + columns = getTableStructureFromFileDescriptor(args.getContext()); + else + { + columns = getTableStructureFromFile(format_name, paths, compression_method, format_settings, args.getContext()); + if (!args.columns.empty() && args.columns != columns) + throw Exception("Table structure and file structure are different", ErrorCodes::INCOMPATIBLE_COLUMNS); + } storage_metadata.setColumns(columns); } else @@ -371,9 +416,15 @@ public: /// Note: AddingDefaultsBlockInputStream doesn't change header. if (need_path_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_path"}); if (need_file_column) - header.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + header.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_file"}); return header; } @@ -397,11 +448,13 @@ public: ContextPtr context_, UInt64 max_block_size_, FilesInfoPtr files_info_, - ColumnsDescription columns_description_) + ColumnsDescription columns_description_, + std::unique_ptr read_buf_) : SourceWithProgress(getBlockForSource(storage_, metadata_snapshot_, columns_description_, files_info_)) , storage(std::move(storage_)) , metadata_snapshot(metadata_snapshot_) , files_info(std::move(files_info_)) + , read_buf(std::move(read_buf_)) , columns_description(std::move(columns_description_)) , context(context_) , max_block_size(max_block_size_) @@ -443,7 +496,8 @@ public: } } - read_buf = createReadBuffer(current_path, storage->use_table_fd, storage->getName(), storage->table_fd, storage->compression_method, context); + if (!read_buf) + read_buf = createReadBuffer(current_path, storage->use_table_fd, storage->getName(), storage->table_fd, storage->compression_method, context); auto get_block_for_format = [&]() -> Block { @@ -480,7 +534,7 @@ public: /// Enrich with virtual columns. if (files_info->need_path_column) { - auto column = DataTypeString().createColumnConst(num_rows, current_path); + auto column = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, current_path); chunk.addColumn(column->convertToFullColumnIfConst()); } @@ -489,7 +543,7 @@ public: size_t last_slash_pos = current_path.find_last_of('/'); auto file_name = current_path.substr(last_slash_pos + 1); - auto column = DataTypeString().createColumnConst(num_rows, std::move(file_name)); + auto column = DataTypeLowCardinality{std::make_shared()}.createColumnConst(num_rows, std::move(file_name)); chunk.addColumn(column->convertToFullColumnIfConst()); } @@ -588,8 +642,16 @@ Pipe StorageFile::read( return metadata_snapshot->getColumns(); }; + /// In case of reading from fd we have to check whether we have already created + /// the read buffer from it in Storage constructor (for schema inference) or not. + /// If yes, then we should use it in StorageFileSource. Atomic bool flag is needed + /// to prevent data race in case of parallel reads. + std::unique_ptr read_buffer; + if (has_peekable_read_buffer_from_fd.exchange(false)) + read_buffer = std::move(peekable_read_buffer_from_fd); + pipes.emplace_back(std::make_shared( - this_ptr, metadata_snapshot, context, max_block_size, files_info, get_columns_for_format())); + this_ptr, metadata_snapshot, context, max_block_size, files_info, get_columns_for_format(), std::move(read_buffer))); } return Pipe::unitePipes(std::move(pipes)); @@ -804,6 +866,7 @@ SinkToStoragePtr StorageFile::write( { if (path_for_partitioned_write.empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty path for partitioned write"); + fs::create_directories(fs::path(path_for_partitioned_write).parent_path()); return std::make_shared( @@ -830,9 +893,10 @@ SinkToStoragePtr StorageFile::write( path = paths.back(); fs::create_directories(fs::path(path).parent_path()); + std::error_code error_code; if (!context->getSettingsRef().engine_file_truncate_on_insert && !is_path_with_globs && !FormatFactory::instance().checkIfFormatSupportAppend(format_name, context, format_settings) && fs::exists(paths.back()) - && fs::file_size(paths.back()) != 0) + && fs::file_size(paths.back(), error_code) != 0 && !error_code) { if (context->getSettingsRef().engine_file_allow_create_multiple_files) { @@ -1050,8 +1114,7 @@ void registerStorageFile(StorageFactory & factory) NamesAndTypesList StorageFile::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } } diff --git a/src/Storages/StorageFile.h b/src/Storages/StorageFile.h index 8564d93ccc2..ed5431d5e03 100644 --- a/src/Storages/StorageFile.h +++ b/src/Storages/StorageFile.h @@ -71,7 +71,9 @@ public: bool supportsPartitionBy() const override { return true; } - static ColumnsDescription getTableStructureFromData( + ColumnsDescription getTableStructureFromFileDescriptor(ContextPtr context); + + static ColumnsDescription getTableStructureFromFile( const String & format, const std::vector & paths, const String & compression_method, @@ -122,6 +124,12 @@ private: String path_for_partitioned_write; bool is_path_with_globs = false; + + /// These buffers are needed for schema inference when data source + /// is file descriptor. See getTableStructureFromFileDescriptor. + std::unique_ptr read_buffer_from_fd; + std::unique_ptr peekable_read_buffer_from_fd; + std::atomic has_peekable_read_buffer_from_fd = false; }; } diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 5f0bd240f64..5ba1514877a 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -732,8 +732,21 @@ void StorageLog::rename(const String & new_path_to_table_data, const StorageID & renameInMemory(new_table_id); } -void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr, TableExclusiveLockHolder &) +static std::chrono::seconds getLockTimeout(ContextPtr context) { + const Settings & settings = context->getSettingsRef(); + Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); + if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) + lock_timeout = settings.max_execution_time.totalSeconds(); + return std::chrono::seconds{lock_timeout}; +} + +void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr context, TableExclusiveLockHolder &) +{ + WriteLock lock{rwlock, getLockTimeout(context)}; + if (!lock) + throw Exception("Lock timeout exceeded", ErrorCodes::TIMEOUT_EXCEEDED); + disk->clearDirectory(table_path); for (auto & data_file : data_files) @@ -750,16 +763,6 @@ void StorageLog::truncate(const ASTPtr &, const StorageMetadataPtr &, ContextPtr } -static std::chrono::seconds getLockTimeout(ContextPtr context) -{ - const Settings & settings = context->getSettingsRef(); - Int64 lock_timeout = settings.lock_acquire_timeout.totalSeconds(); - if (settings.max_execution_time.totalSeconds() != 0 && settings.max_execution_time.totalSeconds() < lock_timeout) - lock_timeout = settings.max_execution_time.totalSeconds(); - return std::chrono::seconds{lock_timeout}; -} - - Pipe StorageLog::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index 49111e02b11..7c5ef5ac04c 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -135,6 +135,10 @@ QueryProcessingStage::Enum StorageMaterializedView::getQueryProcessingStage( const StorageMetadataPtr &, SelectQueryInfo & query_info) const { + /// TODO: Find a way to support projections for StorageMaterializedView. Why do we use different + /// metadata for materialized view and target table? If they are the same, we can get rid of all + /// converting and use it just like a normal view. + query_info.ignore_projections = true; return getTargetTable()->getQueryProcessingStage(local_context, to_stage, getTargetTable()->getInMemoryMetadataPtr(), query_info); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 0dc6f2931d3..433fdb5b0b5 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -188,6 +188,8 @@ QueryProcessingStage::Enum StorageMerge::getQueryProcessingStage( size_t selected_table_size = 0; + /// TODO: Find a way to support projections for StorageMerge + query_info.ignore_projections = true; for (const auto & iterator : database_table_iterators) { while (iterator->isValid()) @@ -471,7 +473,9 @@ Pipe StorageMerge::createSources( modified_context->setSetting("max_threads", streams_num); modified_context->setSetting("max_streams_to_max_threads_ratio", 1); - InterpreterSelectQuery interpreter{modified_query_info.query, modified_context, SelectQueryOptions(processed_stage)}; + /// TODO: Find a way to support projections for StorageMerge + InterpreterSelectQuery interpreter{ + modified_query_info.query, modified_context, SelectQueryOptions(processed_stage).ignoreProjections()}; pipe = QueryPipelineBuilder::getPipe(interpreter.buildQueryPipeline()); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 49040ecbeb3..2e1a11b7ef1 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -2,6 +2,8 @@ #include +#include + #include #include #include @@ -743,9 +745,8 @@ std::shared_ptr StorageMergeTree::selectPartsToMerge( { while (true) { - UInt64 disk_space = getStoragePolicy()->getMaxUnreservedFreeSpace(); select_decision = merger_mutator.selectAllPartsToMergeWithinPartition( - future_part, disk_space, can_merge, partition_id, final, metadata_snapshot, out_disable_reason, optimize_skip_merged_partitions); + future_part, can_merge, partition_id, final, metadata_snapshot, out_disable_reason, optimize_skip_merged_partitions); auto timeout_ms = getSettings()->lock_acquire_timeout_for_background_operations.totalMilliseconds(); auto timeout = std::chrono::milliseconds(timeout_ms); @@ -1184,7 +1185,7 @@ std::vector StorageMergeTree::getSortedPa getUpdatedDataVersion(part, currently_processing_in_background_mutex_lock), part->name }); - std::sort(part_versions_with_names.begin(), part_versions_with_names.end()); + ::sort(part_versions_with_names.begin(), part_versions_with_names.end()); return part_versions_with_names; } @@ -1228,7 +1229,7 @@ bool StorageMergeTree::optimize( constexpr const char * message = "Cannot OPTIMIZE table: {}"; if (disable_reason.empty()) disable_reason = "unknown reason"; - LOG_INFO(log, message, disable_reason); + LOG_INFO(log, fmt::runtime(message), disable_reason); if (local_context->getSettingsRef().optimize_throw_if_noop) throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason); @@ -1254,7 +1255,7 @@ bool StorageMergeTree::optimize( constexpr const char * message = "Cannot OPTIMIZE table: {}"; if (disable_reason.empty()) disable_reason = "unknown reason"; - LOG_INFO(log, message, disable_reason); + LOG_INFO(log, fmt::runtime(message), disable_reason); if (local_context->getSettingsRef().optimize_throw_if_noop) throw Exception(ErrorCodes::CANNOT_ASSIGN_OPTIMIZE, message, disable_reason); diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 304f84c02eb..894b470ef22 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include @@ -37,6 +38,8 @@ public: const StorageMetadataPtr &, SelectQueryInfo & info) const override { + /// TODO: Find a way to support projections for StorageProxy + info.ignore_projections = true; return getNested()->getQueryProcessingStage(context, to_stage, getNested()->getInMemoryMetadataPtr(), info); } diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index a580457fcb6..451238096f4 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -10,6 +10,8 @@ #include #include +#include + #include #include #include @@ -192,6 +194,16 @@ zkutil::ZooKeeperPtr StorageReplicatedMergeTree::getZooKeeper() const return res; } +zkutil::ZooKeeperPtr StorageReplicatedMergeTree::getZooKeeperAndAssertNotReadonly() const +{ + /// There's a short period of time after connection loss when new session is created, + /// but replication queue is not reinitialized. We must ensure that table is not readonly anymore + /// before using new ZooKeeper session to write something (except maybe GET_PART) into replication log. + auto res = getZooKeeper(); + assertNotReadonly(); + return res; +} + static MergeTreePartInfo makeDummyDropRangeForMovePartitionOrAttachPartitionFrom(const String & partition_id) { /// NOTE We don't have special log entry type for MOVE PARTITION/ATTACH PARTITION FROM, @@ -333,14 +345,13 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( /// Do not activate the replica. It will be readonly. LOG_ERROR(log, "No ZooKeeper: table will be in readonly mode."); - is_readonly = true; + has_metadata_in_zookeeper = std::nullopt; return; } if (attach && !current_zookeeper->exists(zookeeper_path + "/metadata")) { LOG_WARNING(log, "No metadata in ZooKeeper for {}: table will be in readonly mode.", zookeeper_path); - is_readonly = true; has_metadata_in_zookeeper = false; return; } @@ -352,11 +363,12 @@ StorageReplicatedMergeTree::StorageReplicatedMergeTree( if (attach && !current_zookeeper->exists(replica_path)) { LOG_WARNING(log, "No metadata in ZooKeeper for {}: table will be in readonly mode", replica_path); - is_readonly = true; has_metadata_in_zookeeper = false; return; } + has_metadata_in_zookeeper = true; + if (!attach) { if (!getDataParts().empty()) @@ -1209,7 +1221,7 @@ void StorageReplicatedMergeTree::checkParts(bool skip_sanity_checks) if (unexpected_parts_nonnew_rows > 0) { - LOG_WARNING(log, sanity_report_fmt, getStorageID().getNameForLogs(), + LOG_WARNING(log, fmt::runtime(sanity_report_fmt), getStorageID().getNameForLogs(), formatReadableQuantity(unexpected_parts_rows), formatReadableQuantity(total_rows_on_filesystem), unexpected_parts.size(), unexpected_parts_rows, unexpected_parts_nonnew, unexpected_parts_nonnew_rows, parts_to_fetch.size(), parts_to_fetch_blocks); @@ -1278,6 +1290,7 @@ void StorageReplicatedMergeTree::checkPartChecksumsAndAddCommitOps(const zkutil: { String columns_str; String checksums_str; + if (zookeeper->tryGet(fs::path(current_part_path) / "columns", columns_str) && zookeeper->tryGet(fs::path(current_part_path) / "checksums", checksums_str)) { @@ -2201,7 +2214,7 @@ void StorageReplicatedMergeTree::executeClonePartFromShard(const LogEntry & entr if (replica.empty()) throw Exception(ErrorCodes::NO_REPLICA_HAS_PART, "Not found active replica on shard {} to clone part {}", entry.source_shard, entry.new_part_name); - LOG_INFO(log, "Will clone part from shard " + entry.source_shard + " and replica " + replica); + LOG_INFO(log, "Will clone part from shard {} and replica {}", entry.source_shard, replica); MutableDataPartPtr part; @@ -2304,7 +2317,7 @@ void StorageReplicatedMergeTree::cloneReplica(const String & source_replica, Coo } } - std::sort(source_queue_names.begin(), source_queue_names.end()); + ::sort(source_queue_names.begin(), source_queue_names.end()); struct QueueEntryInfo { @@ -2774,7 +2787,7 @@ void StorageReplicatedMergeTree::queueUpdatingTask() } try { - queue.pullLogsToQueue(getZooKeeper(), queue_updating_task->getWatchCallback(), ReplicatedMergeTreeQueue::UPDATE); + queue.pullLogsToQueue(getZooKeeperAndAssertNotReadonly(), queue_updating_task->getWatchCallback(), ReplicatedMergeTreeQueue::UPDATE); last_queue_update_finish_time.store(time(nullptr)); queue_update_in_progress = false; } @@ -2859,17 +2872,17 @@ bool StorageReplicatedMergeTree::processQueueEntry(ReplicatedMergeTreeQueue::Sel if (e.code() == ErrorCodes::NO_REPLICA_HAS_PART) { /// If no one has the right part, probably not all replicas work; We will not write to log with Error level. - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); } else if (e.code() == ErrorCodes::ABORTED) { /// Interrupted merge or downloading a part is not an error. - LOG_INFO(log, e.message()); + LOG_INFO(log, fmt::runtime(e.message())); } else if (e.code() == ErrorCodes::PART_IS_TEMPORARILY_LOCKED) { /// Part cannot be added temporarily - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); cleanup_thread.wakeup(); } else @@ -2983,7 +2996,7 @@ void StorageReplicatedMergeTree::mergeSelectingTask() /// (OPTIMIZE queries) can assign new merges. std::lock_guard merge_selecting_lock(merge_selecting_mutex); - auto zookeeper = getZooKeeper(); + auto zookeeper = getZooKeeperAndAssertNotReadonly(); ReplicatedMergeTreeMergePredicate merge_pred = queue.getMergePredicate(zookeeper); @@ -3084,7 +3097,7 @@ void StorageReplicatedMergeTree::mutationsFinalizingTask() try { - needs_reschedule = queue.tryFinalizeMutations(getZooKeeper()); + needs_reschedule = queue.tryFinalizeMutations(getZooKeeperAndAssertNotReadonly()); } catch (...) { @@ -3366,7 +3379,6 @@ void StorageReplicatedMergeTree::startBeingLeader() LOG_INFO(log, "Became leader"); is_leader = true; - merge_selecting_task->activateAndSchedule(); } void StorageReplicatedMergeTree::stopBeingLeader() @@ -3376,7 +3388,6 @@ void StorageReplicatedMergeTree::stopBeingLeader() LOG_INFO(log, "Stopped being leader"); is_leader = false; - merge_selecting_task->deactivate(); } ConnectionTimeouts StorageReplicatedMergeTree::getFetchPartHTTPTimeouts(ContextPtr local_context) @@ -3776,24 +3787,41 @@ bool StorageReplicatedMergeTree::fetchPart(const String & part_name, const Stora if (source_part) { - MinimalisticDataPartChecksums source_part_checksums; - source_part_checksums.computeTotalChecksums(source_part->checksums); + auto source_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksums( + source_part->getColumns(), source_part->checksums); - MinimalisticDataPartChecksums desired_checksums; String part_path = fs::path(source_replica_path) / "parts" / part_name; String part_znode = zookeeper->get(part_path); + std::optional desired_part_header; if (!part_znode.empty()) - desired_checksums = ReplicatedMergeTreePartHeader::fromString(part_znode).getChecksums(); + { + desired_part_header = ReplicatedMergeTreePartHeader::fromString(part_znode); + } else { - String desired_checksums_str = zookeeper->get(fs::path(part_path) / "checksums"); - desired_checksums = MinimalisticDataPartChecksums::deserializeFrom(desired_checksums_str); + String columns_str; + String checksums_str; + + if (zookeeper->tryGet(fs::path(part_path) / "columns", columns_str) && + zookeeper->tryGet(fs::path(part_path) / "checksums", checksums_str)) + { + desired_part_header = ReplicatedMergeTreePartHeader::fromColumnsAndChecksumsZNodes(columns_str, checksums_str); + } + else + { + LOG_INFO(log, "Not checking checksums of part {} with replica {} because part was removed from ZooKeeper", part_name, source_replica_path); + } } - if (source_part_checksums == desired_checksums) + /// Checking both checksums and columns hash. For example we can have empty part + /// with same checksums but different columns. And we attaching it exception will + /// be thrown. + if (desired_part_header + && source_part_header.getColumnsHash() == desired_part_header->getColumnsHash() + && source_part_header.getChecksums() == desired_part_header->getChecksums()) { - LOG_TRACE(log, "Found local part {} with the same checksums as {}", source_part->name, part_name); + LOG_TRACE(log, "Found local part {} with the same checksums and columns hash as {}", source_part->name, part_name); part_to_clone = source_part; } } @@ -4029,7 +4057,8 @@ bool StorageReplicatedMergeTree::fetchExistsPart(const String & part_name, const void StorageReplicatedMergeTree::startup() { - if (is_readonly) + /// Do not start replication if ZooKeeper is not configured or there is no metadata in zookeeper + if (!has_metadata_in_zookeeper.has_value() || !*has_metadata_in_zookeeper) return; try @@ -4039,15 +4068,17 @@ void StorageReplicatedMergeTree::startup() assert(prev_ptr == nullptr); getContext()->getInterserverIOHandler().addEndpoint(data_parts_exchange_ptr->getId(replica_path), data_parts_exchange_ptr); + startBeingLeader(); + /// In this thread replica will be activated. restarting_thread.start(); - /// Wait while restarting_thread finishing initialization + /// Wait while restarting_thread finishing initialization. + /// NOTE It does not mean that replication is actually started after receiving this event. + /// It only means that an attempt to startup replication was made. + /// Table may be still in readonly mode if this attempt failed for any reason. startup_event.wait(); - /// Restarting thread has initialized replication queue, replica can become leader now - startBeingLeader(); - startBackgroundMovesIfNeeded(); part_moves_between_shards_orchestrator.start(); @@ -4335,8 +4366,7 @@ bool StorageReplicatedMergeTree::optimize( return false; }; - auto zookeeper = getZooKeeper(); - UInt64 disk_space = getStoragePolicy()->getMaxUnreservedFreeSpace(); + auto zookeeper = getZooKeeperAndAssertNotReadonly(); const auto storage_settings_ptr = getSettings(); auto metadata_snapshot = getInMemoryMetadataPtr(); std::vector merge_entries; @@ -4369,7 +4399,7 @@ bool StorageReplicatedMergeTree::optimize( else { select_decision = merger_mutator.selectAllPartsToMergeWithinPartition( - future_merged_part, disk_space, can_merge, partition_id, final, metadata_snapshot, + future_merged_part, can_merge, partition_id, final, metadata_snapshot, &disable_reason, query_context->getSettingsRef().optimize_skip_merged_partitions); } @@ -4384,7 +4414,7 @@ bool StorageReplicatedMergeTree::optimize( if (!partition_id.empty()) disable_reason += fmt::format(" (in partition {})", partition_id); String message = fmt::format(message_fmt, disable_reason); - LOG_INFO(log, message); + LOG_INFO(log, fmt::runtime(message)); return handle_noop(message); } @@ -4398,7 +4428,7 @@ bool StorageReplicatedMergeTree::optimize( if (create_result == CreateMergeEntryResult::MissingPart) { String message = "Can't create merge queue node in ZooKeeper, because some parts are missing"; - LOG_TRACE(log, message); + LOG_TRACE(log, fmt::runtime(message)); return handle_noop(message); } @@ -4411,7 +4441,7 @@ bool StorageReplicatedMergeTree::optimize( assert(try_no == max_retries); String message = fmt::format("Can't create merge queue node in ZooKeeper, because log was updated in every of {} tries", try_no); - LOG_TRACE(log, message); + LOG_TRACE(log, fmt::runtime(message)); return handle_noop(message); }; @@ -4479,7 +4509,7 @@ bool StorageReplicatedMergeTree::executeMetadataAlter(const StorageReplicatedMer auto alter_lock_holder = lockForAlter(getSettings()->lock_acquire_timeout_for_background_operations); LOG_INFO(log, "Metadata changed in ZooKeeper. Applying changes locally."); - auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, getInMemoryMetadataPtr()).checkAndFindDiff(metadata_from_entry); + auto metadata_diff = ReplicatedMergeTreeTableMetadata(*this, getInMemoryMetadataPtr()).checkAndFindDiff(metadata_from_entry, getInMemoryMetadataPtr()->getColumns(), getContext()); setTableStructure(std::move(columns_from_entry), metadata_diff); metadata_version = entry.alter_version; @@ -4554,7 +4584,7 @@ void StorageReplicatedMergeTree::alter( return queryToString(query); }; - const auto zookeeper = getZooKeeper(); + const auto zookeeper = getZooKeeperAndAssertNotReadonly(); std::optional alter_entry; std::optional mutation_znode; @@ -4858,7 +4888,6 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper() LOG_INFO(log, "Created ZK nodes for table"); - is_readonly = false; has_metadata_in_zookeeper = true; if (is_first_replica) @@ -4876,7 +4905,7 @@ void StorageReplicatedMergeTree::dropPartNoWaitNoThrow(const String & part_name) if (!is_leader) throw Exception("DROP PART cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); - zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); LogEntry entry; dropPartImpl(zookeeper, part_name, entry, /*detach=*/ false, /*throw_if_noop=*/ false); @@ -4888,7 +4917,7 @@ void StorageReplicatedMergeTree::dropPart(const String & part_name, bool detach, if (!is_leader) throw Exception("DROP PART cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); - zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); LogEntry entry; dropPartImpl(zookeeper, part_name, entry, detach, /*throw_if_noop=*/ true); @@ -4902,7 +4931,7 @@ void StorageReplicatedMergeTree::dropPartition(const ASTPtr & partition, bool de if (!is_leader) throw Exception("DROP PARTITION cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); - zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); LogEntry entry; String partition_id = getPartitionIDFromQuery(partition, query_context); @@ -4925,7 +4954,7 @@ void StorageReplicatedMergeTree::truncate( if (!is_leader) throw Exception("TRUNCATE cannot be done on this replica because it is not a leader", ErrorCodes::NOT_A_LEADER); - zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); Strings partitions = zookeeper->getChildren(fs::path(zookeeper_path) / "block_numbers"); @@ -4949,7 +4978,9 @@ PartitionCommandsResultInfo StorageReplicatedMergeTree::attachPartition( bool attach_part, ContextPtr query_context) { - assertNotReadonly(); + /// Allow ATTACH PARTITION on readonly replica when restoring it. + if (!are_restoring_replica) + assertNotReadonly(); PartitionCommandsResultInfo results; PartsTemporaryRename renamed_parts(*this, "detached/"); @@ -5570,7 +5601,7 @@ void StorageReplicatedMergeTree::fetchPartition( && e.code() != ErrorCodes::CANNOT_READ_ALL_DATA) throw; - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); } return; } @@ -5707,7 +5738,7 @@ void StorageReplicatedMergeTree::fetchPartition( && e.code() != ErrorCodes::CANNOT_READ_ALL_DATA) throw; - LOG_INFO(log, e.displayText()); + LOG_INFO(log, fmt::runtime(e.displayText())); } if (!fetched) @@ -5871,7 +5902,7 @@ CancellationCode StorageReplicatedMergeTree::killMutation(const String & mutatio { assertNotReadonly(); - zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); + zkutil::ZooKeeperPtr zookeeper = getZooKeeperAndAssertNotReadonly(); LOG_INFO(log, "Killing mutation {}", mutation_id); @@ -6610,7 +6641,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( if (zkutil::normalizeZooKeeperPath(zookeeper_path, /* check_starts_with_slash */ true) == zkutil::normalizeZooKeeperPath(to, /* check_starts_with_slash */ true)) throw Exception("Source and destination are the same", ErrorCodes::BAD_ARGUMENTS); - auto zookeeper = getZooKeeper(); + auto zookeeper = getZooKeeperAndAssertNotReadonly(); String part_name = partition->as().value.safeGet(); auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); @@ -6686,7 +6717,7 @@ void StorageReplicatedMergeTree::movePartitionToShard( zkutil::KeeperMultiException::check(rc, ops, responses); String task_znode_path = dynamic_cast(*responses.back()).path_created; - LOG_DEBUG(log, "Created task for part movement between shards at " + task_znode_path); + LOG_DEBUG(log, "Created task for part movement between shards at {}", task_znode_path); /// TODO(nv): Nice to have support for `replication_alter_partitions_sync`. /// For now use the system.part_moves_between_shards table for status. @@ -6796,7 +6827,7 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI Stopwatch watch; /// Let's fetch new log entries firstly - queue.pullLogsToQueue(getZooKeeper(), {}, ReplicatedMergeTreeQueue::SYNC); + queue.pullLogsToQueue(getZooKeeperAndAssertNotReadonly(), {}, ReplicatedMergeTreeQueue::SYNC); /// This is significant, because the execution of this task could be delayed at BackgroundPool. /// And we force it to be executed. @@ -6825,7 +6856,7 @@ bool StorageReplicatedMergeTree::waitForShrinkingQueueSize(size_t queue_size, UI bool StorageReplicatedMergeTree::dropPartImpl( zkutil::ZooKeeperPtr & zookeeper, String part_name, LogEntry & entry, bool detach, bool throw_if_noop) { - LOG_TRACE(log, "Will try to insert a log entry to DROP_RANGE for part: " + part_name); + LOG_TRACE(log, "Will try to insert a log entry to DROP_RANGE for part {}", part_name); auto part_info = MergeTreePartInfo::fromPartName(part_name, format_version); @@ -7028,6 +7059,50 @@ CheckResults StorageReplicatedMergeTree::checkData(const ASTPtr & query, Context } +void StorageReplicatedMergeTree::checkBrokenDisks() +{ + auto disks = getStoragePolicy()->getDisks(); + std::unique_ptr parts; + + for (auto disk_it = disks.rbegin(); disk_it != disks.rend(); ++disk_it) + { + auto disk_ptr = *disk_it; + if (disk_ptr->isBroken()) + { + { + std::unique_lock lock(last_broken_disks_mutex); + if (!last_broken_disks.insert(disk_ptr->getName()).second) + continue; + } + + LOG_INFO(log, "Scanning parts to recover on broken disk {} with path {}", disk_ptr->getName(), disk_ptr->getPath()); + + if (!parts) + parts = std::make_unique(getDataPartsVector()); + + for (auto & part : *parts) + { + if (part->volume && part->volume->getDisk()->getName() == disk_ptr->getName()) + broken_part_callback(part->name); + } + continue; + } + else + { + { + std::unique_lock lock(last_broken_disks_mutex); + if (last_broken_disks.erase(disk_ptr->getName()) > 0) + LOG_INFO( + log, + "Disk {} with path {} is recovered. Exclude it from last_broken_disks", + disk_ptr->getName(), + disk_ptr->getPath()); + } + } + } +} + + bool StorageReplicatedMergeTree::canUseAdaptiveGranularity() const { const auto storage_settings_ptr = getSettings(); @@ -7096,10 +7171,35 @@ void StorageReplicatedMergeTree::createTableSharedID() } -void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) const +void StorageReplicatedMergeTree::lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const { - if (!part.volume) + if (!disk || !disk->supportZeroCopyReplication()) return; + + zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); + if (!zookeeper) + return; + + String id = part_id; + boost::replace_all(id, "/", "_"); + + Strings zc_zookeeper_paths = getZeroCopyPartPath(*getSettings(), disk->getType(), getTableSharedID(), + part_name, zookeeper_path); + + for (const auto & zc_zookeeper_path : zc_zookeeper_paths) + { + String zookeeper_node = fs::path(zc_zookeeper_path) / id / replica_name; + + LOG_TRACE(log, "Set zookeeper temporary ephemeral lock {}", zookeeper_node); + createZeroCopyLockNode(zookeeper, zookeeper_node, zkutil::CreateMode::Ephemeral, false); + } +} + +void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part, bool replace_existing_lock) const +{ + if (!part.volume || !part.isStoredOnDisk()) + return; + DiskPtr disk = part.volume->getDisk(); if (!disk || !disk->supportZeroCopyReplication()) return; @@ -7117,8 +7217,9 @@ void StorageReplicatedMergeTree::lockSharedData(const IMergeTreeDataPart & part) { String zookeeper_node = fs::path(zc_zookeeper_path) / id / replica_name; - LOG_TRACE(log, "Set zookeeper lock {}", zookeeper_node); - createZeroCopyLockNode(zookeeper, zookeeper_node); + LOG_TRACE(log, "Set zookeeper persistent lock {}", zookeeper_node); + + createZeroCopyLockNode(zookeeper, zookeeper_node, zkutil::CreateMode::Persistent, replace_existing_lock); } } @@ -7131,21 +7232,28 @@ bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & par bool StorageReplicatedMergeTree::unlockSharedData(const IMergeTreeDataPart & part, const String & name) const { - if (!part.volume) + if (!part.volume || !part.isStoredOnDisk()) return true; + DiskPtr disk = part.volume->getDisk(); if (!disk || !disk->supportZeroCopyReplication()) return true; - zkutil::ZooKeeperPtr zookeeper = tryGetZooKeeper(); - if (!zookeeper) + /// If part is temporary refcount file may be absent + auto ref_count_path = fs::path(part.getFullRelativePath()) / IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK; + if (disk->exists(ref_count_path)) + { + auto ref_count = disk->getRefCount(ref_count_path); + if (ref_count > 0) /// Keep part shard info for frozen backups + return false; + } + else + { + /// Temporary part with some absent file cannot be locked in shared mode return true; + } - auto ref_count = part.getNumberOfRefereneces(); - if (ref_count > 0) /// Keep part shard info for frozen backups - return false; - - return unlockSharedDataByID(part.getUniqueId(), getTableSharedID(), name, replica_name, disk, zookeeper, *getSettings(), log, + return unlockSharedDataByID(part.getUniqueId(), getTableSharedID(), name, replica_name, disk, getZooKeeper(), *getSettings(), log, zookeeper_path); } @@ -7158,7 +7266,7 @@ bool StorageReplicatedMergeTree::unlockSharedDataByID(String part_id, const Stri Strings zc_zookeeper_paths = getZeroCopyPartPath(settings, disk->getType(), table_uuid, part_name, zookeeper_path_old); - bool res = true; + bool part_has_no_more_locks = true; for (const auto & zc_zookeeper_path : zc_zookeeper_paths) { @@ -7178,7 +7286,7 @@ bool StorageReplicatedMergeTree::unlockSharedDataByID(String part_id, const Stri if (!children.empty()) { LOG_TRACE(logger, "Found zookeper locks for {}", zookeeper_part_uniq_node); - res = false; + part_has_no_more_locks = false; continue; } @@ -7207,7 +7315,7 @@ bool StorageReplicatedMergeTree::unlockSharedDataByID(String part_id, const Stri } } - return res; + return part_has_no_more_locks; } @@ -7329,8 +7437,31 @@ Strings StorageReplicatedMergeTree::getZeroCopyPartPath(const MergeTreeSettings return res; } +bool StorageReplicatedMergeTree::checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk) +{ + auto path = getZeroCopyPartPath(part_name, disk); + if (path) + { + /// FIXME + auto lock_path = fs::path(*path) / "part_exclusive_lock"; + if (getZooKeeper()->exists(lock_path)) + { + return true; + } + } -std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusiveLock(const DataPartPtr & part, const DiskPtr & disk) + return false; +} + +std::optional StorageReplicatedMergeTree::getZeroCopyPartPath(const String & part_name, const DiskPtr & disk) +{ + if (!disk || !disk->supportZeroCopyReplication()) + return std::nullopt; + + return getZeroCopyPartPath(*getSettings(), disk->getType(), getTableSharedID(), part_name, zookeeper_path)[0]; +} + +std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusiveLock(const String & part_name, const DiskPtr & disk) { if (!disk || !disk->supportZeroCopyReplication()) return std::nullopt; @@ -7339,8 +7470,7 @@ std::optional StorageReplicatedMergeTree::tryCreateZeroCopyExclusi if (!zookeeper) return std::nullopt; - String zc_zookeeper_path = getZeroCopyPartPath(*getSettings(), disk->getType(), getTableSharedID(), - part->name, zookeeper_path)[0]; + String zc_zookeeper_path = *getZeroCopyPartPath(part_name, disk); /// Just recursively create ancestors for lock zookeeper->createAncestors(zc_zookeeper_path); @@ -7494,7 +7624,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP /// TODO(ab): What projections should we add to the empty part? How can we make sure that it /// won't block future merges? Perhaps we should also check part emptiness when selecting parts /// to merge. - out.writeSuffixAndFinalizePart(new_data_part, sync_on_insert); + out.finalizePart(new_data_part, sync_on_insert); try { @@ -7575,7 +7705,7 @@ bool StorageReplicatedMergeTree::createEmptyPartInsteadOfLost(zkutil::ZooKeeperP } -void StorageReplicatedMergeTree::createZeroCopyLockNode(const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_node) +void StorageReplicatedMergeTree::createZeroCopyLockNode(const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_node, int32_t mode, bool replace_existing_lock) { /// In rare case other replica can remove path between createAncestors and createIfNotExists /// So we make up to 5 attempts @@ -7585,8 +7715,22 @@ void StorageReplicatedMergeTree::createZeroCopyLockNode(const zkutil::ZooKeeperP try { zookeeper->createAncestors(zookeeper_node); - zookeeper->createIfNotExists(zookeeper_node, "lock"); - break; + if (replace_existing_lock && zookeeper->exists(zookeeper_node)) + { + Coordination::Requests ops; + ops.emplace_back(zkutil::makeRemoveRequest(zookeeper_node, -1)); + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_node, "", mode)); + Coordination::Responses responses; + auto error = zookeeper->tryMulti(ops, responses); + if (error == Coordination::Error::ZOK) + break; + } + else + { + auto error = zookeeper->tryCreate(zookeeper_node, "", mode); + if (error == Coordination::Error::ZOK || error == Coordination::Error::ZNODEEXISTS) + break; + } } catch (const zkutil::KeeperException & e) { @@ -7615,10 +7759,12 @@ public: table_shared_id = storage.getTableSharedID(); } - void save(DiskPtr disk, const String & path) const + void save(DiskPtr data_disk, const String & path) const { + auto metadata_disk = data_disk->getMetadataDiskIfExistsOrSelf(); + auto file_path = getFileName(path); - auto buffer = disk->writeMetaFile(file_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); + auto buffer = metadata_disk->writeFile(file_path, DBMS_DEFAULT_BUFFER_SIZE, WriteMode::Rewrite); writeIntText(version, *buffer); buffer->write("\n", 1); writeBoolText(is_replicated, *buffer); @@ -7633,12 +7779,14 @@ public: buffer->write("\n", 1); } - bool load(DiskPtr disk, const String & path) + bool load(DiskPtr data_disk, const String & path) { + auto metadata_disk = data_disk->getMetadataDiskIfExistsOrSelf(); auto file_path = getFileName(path); - if (!disk->exists(file_path)) + + if (!metadata_disk->exists(file_path)) return false; - auto buffer = disk->readMetaFile(file_path, ReadSettings(), {}); + auto buffer = metadata_disk->readFile(file_path, ReadSettings(), {}); readIntText(version, *buffer); if (version != 1) { @@ -7659,9 +7807,10 @@ public: return true; } - static void clean(DiskPtr disk, const String & path) + static void clean(DiskPtr data_disk, const String & path) { - disk->removeMetaFileIfExists(getFileName(path)); + auto metadata_disk = data_disk->getMetadataDiskIfExistsOrSelf(); + metadata_disk->removeFileIfExists(getFileName(path)); } private: @@ -7715,22 +7864,18 @@ bool StorageReplicatedMergeTree::removeSharedDetachedPart(DiskPtr disk, const St zkutil::ZooKeeperPtr zookeeper = getZooKeeper(); - if (zookeeper) + fs::path checksums = fs::path(path) / IMergeTreeDataPart::FILE_FOR_REFERENCES_CHECK; + if (disk->exists(checksums)) { - fs::path checksums = fs::path(path) / "checksums.txt"; - if (disk->exists(checksums)) + if (disk->getRefCount(checksums) == 0) { - auto ref_count = disk->getRefCount(checksums); - if (ref_count == 0) - { - String id = disk->getUniqueId(checksums); - keep_shared = !StorageReplicatedMergeTree::unlockSharedDataByID(id, table_uuid, part_name, - detached_replica_name, disk, zookeeper, getContext()->getReplicatedMergeTreeSettings(), log, - detached_zookeeper_path); - } - else - keep_shared = true; + String id = disk->getUniqueId(checksums); + keep_shared = !StorageReplicatedMergeTree::unlockSharedDataByID(id, table_uuid, part_name, + detached_replica_name, disk, zookeeper, getContext()->getReplicatedMergeTreeSettings(), log, + detached_zookeeper_path); } + else + keep_shared = true; } disk->removeSharedRecursive(path, keep_shared); diff --git a/src/Storages/StorageReplicatedMergeTree.h b/src/Storages/StorageReplicatedMergeTree.h index c92d76c4062..056671dc164 100644 --- a/src/Storages/StorageReplicatedMergeTree.h +++ b/src/Storages/StorageReplicatedMergeTree.h @@ -231,7 +231,9 @@ public: bool executeFetchShared(const String & source_replica, const String & new_part_name, const DiskPtr & disk, const String & path); /// Lock part in zookeeper for use shared data in several nodes - void lockSharedData(const IMergeTreeDataPart & part) const override; + void lockSharedData(const IMergeTreeDataPart & part, bool replace_existing_lock) const override; + + void lockSharedDataTemporary(const String & part_name, const String & part_id, const DiskPtr & disk) const; /// Unlock shared data part in zookeeper /// Return true if data unlocked @@ -283,6 +285,9 @@ public: static const String getDefaultZooKeeperName() { return default_zookeeper_name; } + /// Check if there are new broken disks and enqueue part recovery tasks. + void checkBrokenDisks(); + private: std::atomic_bool are_restoring_replica {false}; @@ -318,10 +323,12 @@ private: zkutil::ZooKeeperPtr tryGetZooKeeper() const; zkutil::ZooKeeperPtr getZooKeeper() const; + zkutil::ZooKeeperPtr getZooKeeperAndAssertNotReadonly() const; void setZooKeeper(); /// If true, the table is offline and can not be written to it. - std::atomic_bool is_readonly {false}; + /// This flag is managed by RestartingThread. + std::atomic_bool is_readonly {true}; /// If nullopt - ZooKeeper is not available, so we don't know if there is table metadata. /// If false - ZooKeeper is available, but there is no table metadata. It's safe to drop table in this case. std::optional has_metadata_in_zookeeper; @@ -418,6 +425,9 @@ private: /// Global ID, synced via ZooKeeper between replicas UUID table_shared_id; + std::mutex last_broken_disks_mutex; + std::set last_broken_disks; + template void foreachActiveParts(Func && func, bool select_sequential_consistency) const; @@ -750,7 +760,7 @@ private: static Strings getZeroCopyPartPath(const MergeTreeSettings & settings, DiskType disk_type, const String & table_uuid, const String & part_name, const String & zookeeper_path_old); - static void createZeroCopyLockNode(const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_node); + static void createZeroCopyLockNode(const zkutil::ZooKeeperPtr & zookeeper, const String & zookeeper_node, int32_t mode = zkutil::CreateMode::Persistent, bool replace_existing_lock = false); bool removeDetachedPart(DiskPtr disk, const String & path, const String & part_name, bool is_freezed) override; @@ -763,9 +773,14 @@ private: // Create table id if needed void createTableSharedID(); + + bool checkZeroCopyLockExists(const String & part_name, const DiskPtr & disk); + + std::optional getZeroCopyPartPath(const String & part_name, const DiskPtr & disk); + /// Create ephemeral lock in zookeeper for part and disk which support zero copy replication. /// If somebody already holding the lock -- return std::nullopt. - std::optional tryCreateZeroCopyExclusiveLock(const DataPartPtr & part, const DiskPtr & disk) override; + std::optional tryCreateZeroCopyExclusiveLock(const String & part_name, const DiskPtr & disk) override; protected: /** If not 'attach', either creates a new table in ZK, or adds a replica to an existing table. diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index e2294aaabc5..9a85644d825 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -209,9 +209,15 @@ String StorageS3Source::KeysIterator::next() Block StorageS3Source::getHeader(Block sample_block, bool with_path_column, bool with_file_column) { if (with_path_column) - sample_block.insert({DataTypeString().createColumn(), std::make_shared(), "_path"}); + sample_block.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_path"}); if (with_file_column) - sample_block.insert({DataTypeString().createColumn(), std::make_shared(), "_file"}); + sample_block.insert( + {DataTypeLowCardinality{std::make_shared()}.createColumn(), + std::make_shared(std::make_shared()), + "_file"}); return sample_block; } @@ -253,6 +259,7 @@ StorageS3Source::StorageS3Source( void StorageS3Source::onCancel() { + std::lock_guard lock(reader_mutex); if (reader) reader->cancel(); } @@ -295,34 +302,42 @@ String StorageS3Source::getName() const Chunk StorageS3Source::generate() { - if (!reader) - return {}; - - Chunk chunk; - if (reader->pull(chunk)) + while (true) { - UInt64 num_rows = chunk.getNumRows(); + if (!reader || isCancelled()) + break; - if (with_path_column) - chunk.addColumn(DataTypeString().createColumnConst(num_rows, file_path)->convertToFullColumnIfConst()); - if (with_file_column) + Chunk chunk; + if (reader->pull(chunk)) { - size_t last_slash_pos = file_path.find_last_of('/'); - chunk.addColumn(DataTypeString().createColumnConst(num_rows, file_path.substr( - last_slash_pos + 1))->convertToFullColumnIfConst()); + UInt64 num_rows = chunk.getNumRows(); + + if (with_path_column) + chunk.addColumn(DataTypeLowCardinality{std::make_shared()} + .createColumnConst(num_rows, file_path) + ->convertToFullColumnIfConst()); + if (with_file_column) + { + size_t last_slash_pos = file_path.find_last_of('/'); + chunk.addColumn(DataTypeLowCardinality{std::make_shared()} + .createColumnConst(num_rows, file_path.substr(last_slash_pos + 1)) + ->convertToFullColumnIfConst()); + } + + return chunk; } - return chunk; + { + std::lock_guard lock(reader_mutex); + reader.reset(); + pipeline.reset(); + read_buf.reset(); + + if (!initialize()) + break; + } } - - reader.reset(); - pipeline.reset(); - read_buf.reset(); - - if (!initialize()) - return {}; - - return generate(); + return {}; } static bool checkIfObjectExists(const std::shared_ptr & client, const String & bucket, const String & key) @@ -372,13 +387,18 @@ public: const String & bucket, const String & key, size_t min_upload_part_size, + size_t upload_part_size_multiply_factor, + size_t upload_part_size_multiply_parts_count_threshold, size_t max_single_part_upload_size) : SinkToStorage(sample_block_) , sample_block(sample_block_) , format_settings(format_settings_) { write_buf = wrapWriteBufferWithCompressionMethod( - std::make_unique(client, bucket, key, min_upload_part_size, max_single_part_upload_size), compression_method, 3); + std::make_unique( + client, bucket, key, min_upload_part_size, + upload_part_size_multiply_factor, upload_part_size_multiply_parts_count_threshold, + max_single_part_upload_size), compression_method, 3); writer = FormatFactory::instance().getOutputFormatParallelIfPossible(format, *write_buf, sample_block, context, {}, format_settings); } @@ -427,6 +447,8 @@ public: const String & bucket_, const String & key_, size_t min_upload_part_size_, + size_t upload_part_size_multiply_factor_, + size_t upload_part_size_multiply_parts_count_threshold_, size_t max_single_part_upload_size_) : PartitionedSink(partition_by, context_, sample_block_) , format(format_) @@ -437,6 +459,8 @@ public: , bucket(bucket_) , key(key_) , min_upload_part_size(min_upload_part_size_) + , upload_part_size_multiply_factor(upload_part_size_multiply_factor_) + , upload_part_size_multiply_parts_count_threshold(upload_part_size_multiply_parts_count_threshold_) , max_single_part_upload_size(max_single_part_upload_size_) , format_settings(format_settings_) { @@ -460,6 +484,8 @@ public: partition_bucket, partition_key, min_upload_part_size, + upload_part_size_multiply_factor, + upload_part_size_multiply_parts_count_threshold, max_single_part_upload_size ); } @@ -474,6 +500,8 @@ private: const String bucket; const String key; size_t min_upload_part_size; + size_t upload_part_size_multiply_factor; + size_t upload_part_size_multiply_parts_count_threshold; size_t max_single_part_upload_size; std::optional format_settings; @@ -514,6 +542,8 @@ StorageS3::StorageS3( const String & format_name_, UInt64 max_single_read_retries_, UInt64 min_upload_part_size_, + UInt64 upload_part_size_multiply_factor_, + UInt64 upload_part_size_multiply_parts_count_threshold_, UInt64 max_single_part_upload_size_, UInt64 max_connections_, const ColumnsDescription & columns_, @@ -530,6 +560,8 @@ StorageS3::StorageS3( , format_name(format_name_) , max_single_read_retries(max_single_read_retries_) , min_upload_part_size(min_upload_part_size_) + , upload_part_size_multiply_factor(upload_part_size_multiply_factor_) + , upload_part_size_multiply_parts_count_threshold(upload_part_size_multiply_parts_count_threshold_) , max_single_part_upload_size(max_single_part_upload_size_) , compression_method(compression_method_) , name(uri_.storage_name) @@ -656,6 +688,8 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr client_auth.uri.bucket, keys.back(), min_upload_part_size, + upload_part_size_multiply_factor, + upload_part_size_multiply_parts_count_threshold, max_single_part_upload_size); } else @@ -699,6 +733,8 @@ SinkToStoragePtr StorageS3::write(const ASTPtr & query, const StorageMetadataPtr client_auth.uri.bucket, keys.back(), min_upload_part_size, + upload_part_size_multiply_factor, + upload_part_size_multiply_parts_count_threshold, max_single_part_upload_size); } } @@ -910,7 +946,10 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory) S3::URI s3_uri(Poco::URI(configuration.url)); auto max_single_read_retries = args.getLocalContext()->getSettingsRef().s3_max_single_read_retries; auto min_upload_part_size = args.getLocalContext()->getSettingsRef().s3_min_upload_part_size; + auto upload_part_size_multiply_factor = args.getLocalContext()->getSettingsRef().s3_upload_part_size_multiply_factor; + auto upload_part_size_multiply_parts_count_threshold = args.getLocalContext()->getSettingsRef().s3_upload_part_size_multiply_parts_count_threshold; auto max_single_part_upload_size = args.getLocalContext()->getSettingsRef().s3_max_single_part_upload_size; + auto max_connections = args.getLocalContext()->getSettingsRef().s3_max_connections; ASTPtr partition_by; @@ -925,6 +964,8 @@ void registerStorageS3Impl(const String & name, StorageFactory & factory) configuration.format, max_single_read_retries, min_upload_part_size, + upload_part_size_multiply_factor, + upload_part_size_multiply_parts_count_threshold, max_single_part_upload_size, max_connections, args.columns, @@ -957,9 +998,8 @@ void registerStorageCOS(StorageFactory & factory) NamesAndTypesList StorageS3::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } bool StorageS3::supportsPartitionBy() const diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 1b766676427..03b54706b4a 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -99,6 +99,8 @@ private: std::unique_ptr read_buf; std::unique_ptr pipeline; std::unique_ptr reader; + /// onCancel and generate can be called concurrently + std::mutex reader_mutex; bool initialized = false; bool with_file_column = false; bool with_path_column = false; @@ -124,6 +126,8 @@ public: const String & format_name_, UInt64 max_single_read_retries_, UInt64 min_upload_part_size_, + UInt64 upload_part_size_multiply_factor_, + UInt64 upload_part_size_multiply_parts_count_threshold_, UInt64 max_single_part_upload_size_, UInt64 max_connections_, const ColumnsDescription & columns_, @@ -191,6 +195,8 @@ private: String format_name; UInt64 max_single_read_retries; size_t min_upload_part_size; + size_t upload_part_size_multiply_factor; + size_t upload_part_size_multiply_parts_count_threshold; size_t max_single_part_upload_size; String compression_method; String name; diff --git a/src/Storages/StorageS3Cluster.cpp b/src/Storages/StorageS3Cluster.cpp index 659071b392d..57220c68347 100644 --- a/src/Storages/StorageS3Cluster.cpp +++ b/src/Storages/StorageS3Cluster.cpp @@ -82,7 +82,7 @@ Pipe StorageS3Cluster::read( { StorageS3::updateClientAndAuthSettings(context, client_auth); - auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettings()); + auto cluster = context->getCluster(cluster_name)->getClusterWithReplicasAsShards(context->getSettingsRef()); StorageS3::updateClientAndAuthSettings(context, client_auth); auto iterator = std::make_shared(*client_auth.client, client_auth.uri); @@ -152,9 +152,8 @@ QueryProcessingStage::Enum StorageS3Cluster::getQueryProcessingStage( NamesAndTypesList StorageS3Cluster::getVirtuals() const { return NamesAndTypesList{ - {"_path", std::make_shared()}, - {"_file", std::make_shared()} - }; + {"_path", std::make_shared(std::make_shared())}, + {"_file", std::make_shared(std::make_shared())}}; } diff --git a/src/Storages/StorageSQLite.cpp b/src/Storages/StorageSQLite.cpp index 4e2c6cfbe10..f93584ab374 100644 --- a/src/Storages/StorageSQLite.cpp +++ b/src/Storages/StorageSQLite.cpp @@ -40,7 +40,6 @@ StorageSQLite::StorageSQLite( , WithContext(context_->getGlobalContext()) , remote_table_name(remote_table_name_) , database_path(database_path_) - , global_context(context_) , sqlite_db(sqlite_db_) , log(&Poco::Logger::get("StorageSQLite (" + table_id_.table_name + ")")) { diff --git a/src/Storages/StorageSQLite.h b/src/Storages/StorageSQLite.h index dcc17322a1a..e8fd0771ff4 100644 --- a/src/Storages/StorageSQLite.h +++ b/src/Storages/StorageSQLite.h @@ -48,7 +48,6 @@ public: private: String remote_table_name; String database_path; - ContextPtr global_context; SQLitePtr sqlite_db; Poco::Logger * log; }; diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 68c21caf4ab..508c9d8b157 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -36,6 +36,14 @@ namespace ErrorCodes extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NETWORK_ERROR; extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; +} + + +static bool urlWithGlobs(const String & uri) +{ + return (uri.find('{') != std::string::npos && uri.find('}') != std::string::npos) + || uri.find('|') != std::string::npos; } @@ -62,6 +70,7 @@ IStorageURLBase::IStorageURLBase( , partition_by(partition_by_) { StorageInMemoryMetadata storage_metadata; + if (columns_.empty()) { auto columns = getTableStructureFromData(format_name, uri, compression_method, headers, format_settings, context_); @@ -69,40 +78,12 @@ IStorageURLBase::IStorageURLBase( } else storage_metadata.setColumns(columns_); + storage_metadata.setConstraints(constraints_); storage_metadata.setComment(comment); setInMemoryMetadata(storage_metadata); } -ColumnsDescription IStorageURLBase::getTableStructureFromData( - const String & format, - const String & uri, - const String & compression_method, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, - const std::optional & format_settings, - ContextPtr context) -{ - auto read_buffer_creator = [&]() - { - auto parsed_uri = Poco::URI(uri); - return wrapReadBufferWithCompressionMethod( - std::make_unique( - parsed_uri, - Poco::Net::HTTPRequest::HTTP_GET, - nullptr, - ConnectionTimeouts::getHTTPTimeouts(context), - Poco::Net::HTTPBasicCredentials{}, - context->getSettingsRef().max_http_get_redirects, - DBMS_DEFAULT_BUFFER_SIZE, - context->getReadSettings(), - headers, - ReadWriteBufferFromHTTP::Range{}, - context->getRemoteHostFilter()), - chooseCompressionMethod(parsed_uri.getPath(), compression_method)); - }; - - return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); -} namespace { @@ -146,10 +127,25 @@ namespace void onCancel() override { + std::lock_guard lock(reader_mutex); if (reader) reader->cancel(); } + static void setCredentials(Poco::Net::HTTPBasicCredentials & credentials, const Poco::URI & request_uri) + { + const auto & user_info = request_uri.getUserInfo(); + if (!user_info.empty()) + { + std::size_t n = user_info.find(':'); + if (n != std::string::npos) + { + credentials.setUsername(user_info.substr(0, n)); + credentials.setPassword(user_info.substr(n + 1)); + } + } + } + StorageURLSource( URIInfoPtr uri_info_, const std::string & http_method, @@ -164,7 +160,8 @@ namespace const ConnectionTimeouts & timeouts, const String & compression_method, const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_ = {}, - const URIParams & params = {}) + const URIParams & params = {}, + bool glob_url = false) : SourceWithProgress(sample_block), name(std::move(name_)) , uri_info(uri_info_) { @@ -173,53 +170,43 @@ namespace /// Lazy initialization. We should not perform requests in constructor, because we need to do it in query pipeline. initialize = [=, this](const URIInfo::FailoverOptions & uri_options) { - WriteBufferFromOwnString error_message; - for (auto option = uri_options.begin(); option < uri_options.end(); ++option) + if (uri_options.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got empty url list"); + + if (uri_options.size() > 1) { - auto request_uri = Poco::URI(*option); + read_buf = getFirstAvailableURLReadBuffer( + uri_options, context, params, http_method, + callback, timeouts, compression_method, credentials, headers); + } + else + { + ReadSettings read_settings = context->getReadSettings(); + bool skip_url_not_found_error = glob_url && read_settings.http_skip_not_found_url_for_globs; + auto request_uri = Poco::URI(uri_options[0]); + for (const auto & [param, value] : params) request_uri.addQueryParameter(param, value); - try - { - std::string user_info = request_uri.getUserInfo(); - if (!user_info.empty()) - { - std::size_t n = user_info.find(':'); - if (n != std::string::npos) - { - credentials.setUsername(user_info.substr(0, n)); - credentials.setPassword(user_info.substr(n + 1)); - } - } + setCredentials(credentials, request_uri); - /// Get first alive uri. - read_buf = wrapReadBufferWithCompressionMethod( - std::make_unique( - request_uri, - http_method, - callback, - timeouts, - credentials, - context->getSettingsRef().max_http_get_redirects, - DBMS_DEFAULT_BUFFER_SIZE, - context->getReadSettings(), - headers, - ReadWriteBufferFromHTTP::Range{}, - context->getRemoteHostFilter()), - chooseCompressionMethod(request_uri.getPath(), compression_method)); - } - catch (...) - { - if (uri_options.size() == 1) - throw; - - if (option == uri_options.end() - 1) - throw Exception(ErrorCodes::NETWORK_ERROR, "All uri options are unreachable. {}", error_message.str()); - - error_message << *option << " error: " << getCurrentExceptionMessage(false) << "\n"; - tryLogCurrentException(__PRETTY_FUNCTION__); - } + read_buf = wrapReadBufferWithCompressionMethod( + std::make_unique( + request_uri, + http_method, + callback, + timeouts, + credentials, + context->getSettingsRef().max_http_get_redirects, + DBMS_DEFAULT_BUFFER_SIZE, + read_settings, + headers, + ReadWriteBufferFromHTTP::Range{}, + context->getRemoteHostFilter(), + /* delay_initiliazation */true, + /* use_external_buffer */false, + /* skip_url_not_found_error */skip_url_not_found_error), + chooseCompressionMethod(request_uri.getPath(), compression_method)); } auto input_format = FormatFactory::instance().getInput(format, *read_buf, sample_block, context, max_block_size, format_settings); @@ -245,6 +232,7 @@ namespace { while (true) { + if (!reader) { auto current_uri_pos = uri_info->next_uri_to_read.fetch_add(1); @@ -252,6 +240,8 @@ namespace return {}; auto current_uri = uri_info->uri_list_to_read[current_uri_pos]; + + std::lock_guard lock(reader_mutex); initialize(current_uri); } @@ -259,11 +249,73 @@ namespace if (reader->pull(chunk)) return chunk; - pipeline->reset(); - reader.reset(); + { + std::lock_guard lock(reader_mutex); + pipeline->reset(); + reader.reset(); + } } } + static std::unique_ptr getFirstAvailableURLReadBuffer( + const std::vector & urls, + ContextPtr context, + const URIParams & params, + const String & http_method, + std::function callback, + const ConnectionTimeouts & timeouts, + const String & compression_method, + Poco::Net::HTTPBasicCredentials & credentials, + const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers) + { + String first_exception_message; + ReadSettings read_settings = context->getReadSettings(); + + for (auto option = urls.begin(); option != urls.end(); ++option) + { + bool skip_url_not_found_error = read_settings.http_skip_not_found_url_for_globs && option == std::prev(urls.end()); + auto request_uri = Poco::URI(*option); + + for (const auto & [param, value] : params) + request_uri.addQueryParameter(param, value); + + setCredentials(credentials, request_uri); + + try + { + return wrapReadBufferWithCompressionMethod( + std::make_unique( + request_uri, + http_method, + callback, + timeouts, + credentials, + context->getSettingsRef().max_http_get_redirects, + DBMS_DEFAULT_BUFFER_SIZE, + read_settings, + headers, + ReadWriteBufferFromHTTP::Range{}, + context->getRemoteHostFilter(), + /* delay_initiliazation */false, + /* use_external_buffer */false, + /* skip_url_not_found_error */skip_url_not_found_error), + chooseCompressionMethod(request_uri.getPath(), compression_method)); + } + catch (...) + { + if (first_exception_message.empty()) + first_exception_message = getCurrentExceptionMessage(false); + + if (urls.size() == 1) + throw; + + tryLogCurrentException(__PRETTY_FUNCTION__); + } + } + + throw Exception(ErrorCodes::NETWORK_ERROR, "All uri ({}) options are unreachable: {}", urls.size(), first_exception_message); + } + private: using InitializeFunc = std::function; InitializeFunc initialize; @@ -274,8 +326,11 @@ namespace std::unique_ptr read_buf; std::unique_ptr pipeline; std::unique_ptr reader; + /// onCancell and generate can be called concurrently and both of them + /// have R/W access to reader pointer. + std::mutex reader_mutex; - Poco::Net::HTTPBasicCredentials credentials{}; + Poco::Net::HTTPBasicCredentials credentials; }; } @@ -291,9 +346,10 @@ StorageURLSink::StorageURLSink( : SinkToStorage(sample_block) { std::string content_type = FormatFactory::instance().getContentType(format, context, format_settings); + std::string content_encoding = toContentEncodingName(compression_method); write_buf = wrapWriteBufferWithCompressionMethod( - std::make_unique(Poco::URI(uri), http_method, content_type, timeouts), + std::make_unique(Poco::URI(uri), http_method, content_type, content_encoding, timeouts), compression_method, 3); writer = FormatFactory::instance().getOutputFormat(format, *write_buf, sample_block, context, {} /* write callback */, format_settings); @@ -385,6 +441,71 @@ std::function IStorageURLBase::getReadPOSTDataCallback( } +ColumnsDescription IStorageURLBase::getTableStructureFromData( + const String & format, + const String & uri, + const String & compression_method, + const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + const std::optional & format_settings, + ContextPtr context) +{ + ReadBufferCreator read_buffer_creator; + Poco::Net::HTTPBasicCredentials credentials; + + if (urlWithGlobs(uri)) + { + std::vector urls_to_check; + + size_t max_addresses = context->getSettingsRef().glob_expansion_max_elements; + auto uri_descriptions = parseRemoteDescription(uri, 0, uri.size(), ',', max_addresses); + for (const auto & description : uri_descriptions) + { + auto options = parseRemoteDescription(description, 0, description.size(), '|', max_addresses); + urls_to_check.insert(urls_to_check.end(), options.begin(), options.end()); + } + + read_buffer_creator = [&, urls_to_check]() + { + return StorageURLSource::getFirstAvailableURLReadBuffer( + urls_to_check, + context, + {}, + Poco::Net::HTTPRequest::HTTP_GET, + {}, + ConnectionTimeouts::getHTTPTimeouts(context), + compression_method, + credentials, + headers); + }; + } + else + { + read_buffer_creator = [&]() + { + auto parsed_uri = Poco::URI(uri); + StorageURLSource::setCredentials(credentials, parsed_uri); + + return wrapReadBufferWithCompressionMethod( + std::make_unique( + parsed_uri, + Poco::Net::HTTPRequest::HTTP_GET, + nullptr, + ConnectionTimeouts::getHTTPTimeouts(context), + credentials, + context->getSettingsRef().max_http_get_redirects, + DBMS_DEFAULT_BUFFER_SIZE, + context->getReadSettings(), + headers, + ReadWriteBufferFromHTTP::Range{}, + context->getRemoteHostFilter(), + /* delay_initiliazation */true), + chooseCompressionMethod(parsed_uri.getPath(), compression_method)); + }; + } + + return readSchemaFromFormat(format, format_settings, read_buffer_creator, context); +} + Pipe IStorageURLBase::read( const Names & column_names, const StorageMetadataPtr & metadata_snapshot, @@ -395,10 +516,8 @@ Pipe IStorageURLBase::read( unsigned num_streams) { auto params = getReadURIParams(column_names, metadata_snapshot, query_info, local_context, processed_stage, max_block_size); - bool with_globs = (uri.find('{') != std::string::npos && uri.find('}') != std::string::npos) - || uri.find('|') != std::string::npos; - if (with_globs) + if (urlWithGlobs(uri)) { size_t max_addresses = local_context->getSettingsRef().glob_expansion_max_elements; auto uri_descriptions = parseRemoteDescription(uri, 0, uri.size(), ',', max_addresses); @@ -430,7 +549,7 @@ Pipe IStorageURLBase::read( metadata_snapshot->getColumns(), max_block_size, ConnectionTimeouts::getHTTPTimeouts(local_context), - compression_method, headers, params)); + compression_method, headers, params, /* glob_url */true)); } return Pipe::unitePipes(std::move(pipes)); } diff --git a/src/Storages/System/StorageSystemAsynchronousInserts.h b/src/Storages/System/StorageSystemAsynchronousInserts.h index 79f19ec3d97..d25217006db 100644 --- a/src/Storages/System/StorageSystemAsynchronousInserts.h +++ b/src/Storages/System/StorageSystemAsynchronousInserts.h @@ -14,7 +14,7 @@ class StorageSystemAsynchronousInserts final : public IStorageSystemOneBlock { public: - std::string getName() const override { return "AsynchronousInserts"; } + std::string getName() const override { return "SystemAsynchronousInserts"; } static NamesAndTypesList getNamesAndTypes(); protected: diff --git a/src/Storages/System/StorageSystemGrants.cpp b/src/Storages/System/StorageSystemGrants.cpp index f55145ccfc7..26bd241023a 100644 --- a/src/Storages/System/StorageSystemGrants.cpp +++ b/src/Storages/System/StorageSystemGrants.cpp @@ -23,7 +23,7 @@ NamesAndTypesList StorageSystemGrants::getNamesAndTypes() NamesAndTypesList names_and_types{ {"user_name", std::make_shared(std::make_shared())}, {"role_name", std::make_shared(std::make_shared())}, - {"access_type", std::make_shared(StorageSystemPrivileges::getAccessTypeEnumValues())}, + {"access_type", std::make_shared(StorageSystemPrivileges::getAccessTypeEnumValues())}, {"database", std::make_shared(std::make_shared())}, {"table", std::make_shared(std::make_shared())}, {"column", std::make_shared(std::make_shared())}, @@ -46,7 +46,7 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, ContextPtr cont auto & column_user_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); auto & column_role_name = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); auto & column_role_name_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); - auto & column_access_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_access_type = assert_cast(*res_columns[column_index++]).getData(); auto & column_database = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); auto & column_database_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); auto & column_table = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()); @@ -82,7 +82,7 @@ void StorageSystemGrants::fillData(MutableColumns & res_columns, ContextPtr cont else assert(false); - column_access_type.push_back(static_cast(access_type)); + column_access_type.push_back(static_cast(access_type)); if (database) { diff --git a/src/Storages/System/StorageSystemPartsBase.cpp b/src/Storages/System/StorageSystemPartsBase.cpp index 6c8159ca720..f4dd9cbd45d 100644 --- a/src/Storages/System/StorageSystemPartsBase.cpp +++ b/src/Storages/System/StorageSystemPartsBase.cpp @@ -66,7 +66,7 @@ StoragesInfo::getParts(MergeTreeData::DataPartStateVector & state, bool has_stat } StoragesInfoStream::StoragesInfoStream(const SelectQueryInfo & query_info, ContextPtr context) - : query_id(context->getCurrentQueryId()), settings(context->getSettings()) + : query_id(context->getCurrentQueryId()), settings(context->getSettingsRef()) { /// Will apply WHERE to subset of columns and then add more columns. /// This is kind of complicated, but we use WHERE to do less work. diff --git a/src/Storages/System/StorageSystemPrivileges.cpp b/src/Storages/System/StorageSystemPrivileges.cpp index 6a4d2e1087e..8cf1accfe34 100644 --- a/src/Storages/System/StorageSystemPrivileges.cpp +++ b/src/Storages/System/StorageSystemPrivileges.cpp @@ -44,11 +44,11 @@ namespace } -const std::vector> & StorageSystemPrivileges::getAccessTypeEnumValues() +const std::vector> & StorageSystemPrivileges::getAccessTypeEnumValues() { - static const std::vector> values = [] + static const std::vector> values = [] { - std::vector> res; + std::vector> res; #define ADD_ACCESS_TYPE_ENUM_VALUE(name, aliases, node_type, parent_group_name) \ res.emplace_back(toString(AccessType::name), static_cast(AccessType::name)); @@ -65,10 +65,10 @@ const std::vector> & StorageSystemPrivileges::getAccessT NamesAndTypesList StorageSystemPrivileges::getNamesAndTypes() { NamesAndTypesList names_and_types{ - {"privilege", std::make_shared(getAccessTypeEnumValues())}, + {"privilege", std::make_shared(getAccessTypeEnumValues())}, {"aliases", std::make_shared(std::make_shared())}, {"level", std::make_shared(std::make_shared(getLevelEnumValues()))}, - {"parent_group", std::make_shared(std::make_shared(getAccessTypeEnumValues()))}, + {"parent_group", std::make_shared(std::make_shared(getAccessTypeEnumValues()))}, }; return names_and_types; } @@ -77,17 +77,17 @@ NamesAndTypesList StorageSystemPrivileges::getNamesAndTypes() void StorageSystemPrivileges::fillData(MutableColumns & res_columns, ContextPtr, const SelectQueryInfo &) const { size_t column_index = 0; - auto & column_access_type = assert_cast(*res_columns[column_index++]).getData(); + auto & column_access_type = assert_cast(*res_columns[column_index++]).getData(); auto & column_aliases = assert_cast(assert_cast(*res_columns[column_index]).getData()); auto & column_aliases_offsets = assert_cast(*res_columns[column_index++]).getOffsets(); auto & column_level = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()).getData(); auto & column_level_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); - auto & column_parent_group = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()).getData(); + auto & column_parent_group = assert_cast(assert_cast(*res_columns[column_index]).getNestedColumn()).getData(); auto & column_parent_group_null_map = assert_cast(*res_columns[column_index++]).getNullMapData(); auto add_row = [&](AccessType access_type, const std::string_view & aliases, Level max_level, AccessType parent_group) { - column_access_type.push_back(static_cast(access_type)); + column_access_type.push_back(static_cast(access_type)); for (size_t pos = 0; pos < aliases.length();) { @@ -121,7 +121,7 @@ void StorageSystemPrivileges::fillData(MutableColumns & res_columns, ContextPtr, } else { - column_parent_group.push_back(static_cast(parent_group)); + column_parent_group.push_back(static_cast(parent_group)); column_parent_group_null_map.push_back(false); } }; diff --git a/src/Storages/System/StorageSystemPrivileges.h b/src/Storages/System/StorageSystemPrivileges.h index dad12d14ac0..5eaba9bed79 100644 --- a/src/Storages/System/StorageSystemPrivileges.h +++ b/src/Storages/System/StorageSystemPrivileges.h @@ -14,7 +14,7 @@ class StorageSystemPrivileges final : public shared_ptr_helper> & getAccessTypeEnumValues(); + static const std::vector> & getAccessTypeEnumValues(); protected: friend struct shared_ptr_helper; diff --git a/src/Storages/TTLDescription.cpp b/src/Storages/TTLDescription.cpp index bd5cc9e2f9d..ccf924f2827 100644 --- a/src/Storages/TTLDescription.cpp +++ b/src/Storages/TTLDescription.cpp @@ -15,6 +15,9 @@ #include #include +#include +#include +#include namespace DB @@ -109,6 +112,7 @@ TTLDescription::TTLDescription(const TTLDescription & other) , aggregate_descriptions(other.aggregate_descriptions) , destination_type(other.destination_type) , destination_name(other.destination_name) + , if_exists(other.if_exists) , recompression_codec(other.recompression_codec) { if (other.expression) @@ -146,6 +150,7 @@ TTLDescription & TTLDescription::operator=(const TTLDescription & other) aggregate_descriptions = other.aggregate_descriptions; destination_type = other.destination_type; destination_name = other.destination_name; + if_exists = other.if_exists; if (other.recompression_codec) recompression_codec = other.recompression_codec->clone(); @@ -182,9 +187,10 @@ TTLDescription TTLDescription::getTTLFromAST( } else /// rows TTL { + result.mode = ttl_element->mode; result.destination_type = ttl_element->destination_type; result.destination_name = ttl_element->destination_name; - result.mode = ttl_element->mode; + result.if_exists = ttl_element->if_exists; if (ttl_element->mode == TTLMode::DELETE) { @@ -370,4 +376,17 @@ TTLTableDescription TTLTableDescription::getTTLForTableFromAST( return result; } +TTLTableDescription TTLTableDescription::parse(const String & str, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key) +{ + TTLTableDescription result; + if (str.empty()) + return result; + + ParserTTLExpressionList parser; + ASTPtr ast = parseQuery(parser, str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH); + FunctionNameNormalizer().visit(ast.get()); + + return getTTLForTableFromAST(ast, columns, context, primary_key); +} + } diff --git a/src/Storages/TTLDescription.h b/src/Storages/TTLDescription.h index 6288098b3c5..8f60eb604b5 100644 --- a/src/Storages/TTLDescription.h +++ b/src/Storages/TTLDescription.h @@ -75,6 +75,10 @@ struct TTLDescription /// Name of destination disk or volume String destination_name; + /// If true, do nothing if DISK or VOLUME doesn't exist . + /// Only valid for table MOVE TTLs. + bool if_exists = false; + /// Codec name which will be used to recompress data ASTPtr recompression_codec; @@ -118,6 +122,9 @@ struct TTLTableDescription static TTLTableDescription getTTLForTableFromAST( const ASTPtr & definition_ast, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key); + + /// Parse description from string + static TTLTableDescription parse(const String & str, const ColumnsDescription & columns, ContextPtr context, const KeyDescription & primary_key); }; } diff --git a/src/Storages/registerStorages.cpp b/src/Storages/registerStorages.cpp index d61baf2eb63..f567bf6eefc 100644 --- a/src/Storages/registerStorages.cpp +++ b/src/Storages/registerStorages.cpp @@ -132,11 +132,11 @@ void registerStorages() registerStorageKafka(factory); #endif -#if USE_FILELOG + #if USE_FILELOG registerStorageFileLog(factory); -#endif + #endif -#if USE_AMQPCPP + #if USE_AMQPCPP registerStorageRabbitMQ(factory); #endif diff --git a/src/TableFunctions/TableFunctionFile.cpp b/src/TableFunctions/TableFunctionFile.cpp index 6e288f9fa1e..192846f7f11 100644 --- a/src/TableFunctions/TableFunctionFile.cpp +++ b/src/TableFunctions/TableFunctionFile.cpp @@ -38,7 +38,7 @@ ColumnsDescription TableFunctionFile::getActualTableStructure(ContextPtr context { size_t total_bytes_to_read = 0; Strings paths = StorageFile::getPathsList(filename, context->getUserFilesPath(), context, total_bytes_to_read); - return StorageFile::getTableStructureFromData(format, paths, compression_method, std::nullopt, context); + return StorageFile::getTableStructureFromFile(format, paths, compression_method, std::nullopt, context); } return parseColumnsListFromString(structure, context); diff --git a/src/TableFunctions/TableFunctionFormat.cpp b/src/TableFunctions/TableFunctionFormat.cpp new file mode 100644 index 00000000000..a66edd0003c --- /dev/null +++ b/src/TableFunctions/TableFunctionFormat.cpp @@ -0,0 +1,99 @@ +#include + +#include + +#include +#include + +#include + +#include +#include + +#include +#include + +#include + +#include +#include + + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; +} + +void TableFunctionFormat::parseArguments(const ASTPtr & ast_function, ContextPtr context) +{ + ASTs & args_func = ast_function->children; + + if (args_func.size() != 1) + throw Exception("Table function '" + getName() + "' must have arguments", ErrorCodes::LOGICAL_ERROR); + + ASTs & args = args_func.at(0)->children; + + if (args.size() != 2) + throw Exception("Table function '" + getName() + "' requires 2 arguments: format and data", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + + for (auto & arg : args) + arg = evaluateConstantExpressionOrIdentifierAsLiteral(arg, context); + + format = args[0]->as().value.safeGet(); + data = args[1]->as().value.safeGet(); +} + +ColumnsDescription TableFunctionFormat::getActualTableStructure(ContextPtr context) const +{ + auto read_buffer_creator = [&]() + { + return std::make_unique(data); + }; + + return readSchemaFromFormat(format, std::nullopt, read_buffer_creator, context); +} + +Block TableFunctionFormat::parseData(ColumnsDescription columns, ContextPtr context) const +{ + Block block; + for (const auto & name_and_type : columns.getAllPhysical()) + block.insert({name_and_type.type->createColumn(), name_and_type.type, name_and_type.name}); + + auto read_buf = std::make_unique(data); + auto input_format = context->getInputFormat(format, *read_buf, block, context->getSettingsRef().max_block_size); + QueryPipelineBuilder builder; + builder.init(Pipe(input_format)); + auto pipeline = std::make_unique(QueryPipelineBuilder::getPipeline(std::move(builder))); + auto reader = std::make_unique(*pipeline); + + std::vector blocks; + while (reader->pull(block)) + blocks.push_back(std::move(block)); + + if (blocks.size() == 1) + return blocks[0]; + + /// In case when data contains more then 1 block we combine + /// them all to one big block (this is considered a rare case). + return concatenateBlocks(blocks); +} + +StoragePtr TableFunctionFormat::executeImpl(const ASTPtr & /*ast_function*/, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const +{ + auto columns = getActualTableStructure(context); + Block res_block = parseData(columns, context); + auto res = StorageValues::create(StorageID(getDatabaseName(), table_name), columns, res_block); + res->startup(); + return res; +} + +void registerTableFunctionFormat(TableFunctionFactory & factory) +{ + factory.registerFunction(TableFunctionFactory::CaseInsensitive); +} + +} diff --git a/src/TableFunctions/TableFunctionFormat.h b/src/TableFunctions/TableFunctionFormat.h new file mode 100644 index 00000000000..c6db322343b --- /dev/null +++ b/src/TableFunctions/TableFunctionFormat.h @@ -0,0 +1,33 @@ +#pragma once + +#include + + +namespace DB +{ + +class Context; + +/* format(format_name, data) - ... + */ +class TableFunctionFormat : public ITableFunction +{ +public: + static constexpr auto name = "format"; + std::string getName() const override { return name; } + bool hasStaticStructure() const override { return false; } + +private: + StoragePtr executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns) const override; + const char * getStorageTypeName() const override { return "Values"; } + + ColumnsDescription getActualTableStructure(ContextPtr context) const override; + void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; + + Block parseData(ColumnsDescription columns, ContextPtr context) const; + + String format; + String data; +}; + +} diff --git a/src/TableFunctions/TableFunctionInput.cpp b/src/TableFunctions/TableFunctionInput.cpp index 677a6ff3ce4..cba145ee87b 100644 --- a/src/TableFunctions/TableFunctionInput.cpp +++ b/src/TableFunctions/TableFunctionInput.cpp @@ -5,11 +5,9 @@ #include #include #include -#include #include #include #include -#include #include "registerTableFunctions.h" diff --git a/src/TableFunctions/TableFunctionRemote.cpp b/src/TableFunctions/TableFunctionRemote.cpp index 85857011616..90fbb079bb6 100644 --- a/src/TableFunctions/TableFunctionRemote.cpp +++ b/src/TableFunctions/TableFunctionRemote.cpp @@ -205,7 +205,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr if (name != "clusterAllReplicas") cluster = context->getCluster(cluster_name_expanded); else - cluster = context->getCluster(cluster_name_expanded)->getClusterWithReplicasAsShards(context->getSettings()); + cluster = context->getCluster(cluster_name_expanded)->getClusterWithReplicasAsShards(context->getSettingsRef()); } else { @@ -241,7 +241,7 @@ void TableFunctionRemote::parseArguments(const ASTPtr & ast_function, ContextPtr bool treat_local_as_remote = false; bool treat_local_port_as_remote = context->getApplicationType() == Context::ApplicationType::LOCAL; cluster = std::make_shared( - context->getSettings(), + context->getSettingsRef(), names, configuration.username, configuration.password, diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index be217868c15..1660cadab04 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -29,9 +29,12 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con const auto message = fmt::format( "The signature of table function {} could be the following:\n" \ + " - url\n" " - url, format\n" \ " - url, format, structure\n" \ + " - url, access_key_id, secret_access_key\n" \ " - url, format, structure, compression_method\n" \ + " - url, access_key_id, secret_access_key, format\n" " - url, access_key_id, secret_access_key, format, structure\n" \ " - url, access_key_id, secret_access_key, format, structure, compression_method", getName()); @@ -62,7 +65,7 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con } else { - if (args.size() < 3 || args.size() > 6) + if (args.empty() || args.size() > 6) throw Exception(message, ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); for (auto & arg : args) @@ -73,7 +76,6 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con { {1, {{}}}, {2, {{"format", 1}}}, - {3, {{"format", 1}, {"structure", 2}}}, {5, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}}}, {6, {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}, {"structure", 4}, {"compression_method", 5}}} }; @@ -81,14 +83,26 @@ void TableFunctionS3::parseArguments(const ASTPtr & ast_function, ContextPtr con std::map args_to_idx; /// For 4 arguments we support 2 possible variants: /// s3(source, format, structure, compression_method) and s3(source, access_key_id, access_key_id, format) - /// We can distinguish them by looking at the 4-th argument: check if it's a format name or not. + /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. if (args.size() == 4) { - auto last_arg = args[3]->as().value.safeGet(); - if (FormatFactory::instance().getAllFormats().contains(last_arg)) - args_to_idx = {{"access_key_id", 1}, {"access_key_id", 2}, {"format", 3}}; - else + auto second_arg = args[1]->as().value.safeGet(); + if (FormatFactory::instance().getAllFormats().contains(second_arg)) args_to_idx = {{"format", 1}, {"structure", 2}, {"compression_method", 3}}; + + else + args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}, {"format", 3}}; + } + /// For 3 arguments we support 2 possible variants: + /// s3(source, format, structure) and s3(source, access_key_id, access_key_id) + /// We can distinguish them by looking at the 2-nd argument: check if it's a format name or not. + else if (args.size() == 3) + { + auto second_arg = args[1]->as().value.safeGet(); + if (FormatFactory::instance().getAllFormats().contains(second_arg)) + args_to_idx = {{"format", 1}, {"structure", 2}}; + else + args_to_idx = {{"access_key_id", 1}, {"secret_access_key", 2}}; } else { @@ -146,6 +160,8 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, Context S3::URI s3_uri (uri); UInt64 max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; UInt64 min_upload_part_size = context->getSettingsRef().s3_min_upload_part_size; + UInt64 upload_part_size_multiply_factor = context->getSettingsRef().s3_upload_part_size_multiply_factor; + UInt64 upload_part_size_multiply_parts_count_threshold = context->getSettingsRef().s3_upload_part_size_multiply_parts_count_threshold; UInt64 max_single_part_upload_size = context->getSettingsRef().s3_max_single_part_upload_size; UInt64 max_connections = context->getSettingsRef().s3_max_connections; @@ -161,6 +177,8 @@ StoragePtr TableFunctionS3::executeImpl(const ASTPtr & /*ast_function*/, Context s3_configuration->format, max_single_read_retries, min_upload_part_size, + upload_part_size_multiply_factor, + upload_part_size_multiply_parts_count_threshold, max_single_part_upload_size, max_connections, getActualTableStructure(context), diff --git a/src/TableFunctions/TableFunctionS3Cluster.cpp b/src/TableFunctions/TableFunctionS3Cluster.cpp index aa3ae20b61d..bc215b578b9 100644 --- a/src/TableFunctions/TableFunctionS3Cluster.cpp +++ b/src/TableFunctions/TableFunctionS3Cluster.cpp @@ -109,6 +109,8 @@ StoragePtr TableFunctionS3Cluster::executeImpl( /// Actually this parameters are not used UInt64 max_single_read_retries = context->getSettingsRef().s3_max_single_read_retries; UInt64 min_upload_part_size = context->getSettingsRef().s3_min_upload_part_size; + UInt64 upload_part_size_multiply_factor = context->getSettingsRef().s3_upload_part_size_multiply_factor; + UInt64 upload_part_size_multiply_parts_count_threshold = context->getSettingsRef().s3_upload_part_size_multiply_parts_count_threshold; UInt64 max_single_part_upload_size = context->getSettingsRef().s3_max_single_part_upload_size; UInt64 max_connections = context->getSettingsRef().s3_max_connections; storage = StorageS3::create( @@ -119,6 +121,8 @@ StoragePtr TableFunctionS3Cluster::executeImpl( format, max_single_read_retries, min_upload_part_size, + upload_part_size_multiply_factor, + upload_part_size_multiply_parts_count_threshold, max_single_part_upload_size, max_connections, getActualTableStructure(context), diff --git a/src/TableFunctions/TableFunctionValues.cpp b/src/TableFunctions/TableFunctionValues.cpp index c66ebe7322e..07019d26067 100644 --- a/src/TableFunctions/TableFunctionValues.cpp +++ b/src/TableFunctions/TableFunctionValues.cpp @@ -4,12 +4,11 @@ #include #include #include +#include #include #include -#include -#include #include #include #include @@ -28,13 +27,14 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; + extern const int CANNOT_EXTRACT_TABLE_STRUCTURE; } -static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args, const Block & sample_block, ContextPtr context) +static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args, const Block & sample_block, size_t start, ContextPtr context) { if (res_columns.size() == 1) /// Parsing arguments as Fields { - for (size_t i = 1; i < args.size(); ++i) + for (size_t i = start; i < args.size(); ++i) { const auto & [value_field, value_type_ptr] = evaluateConstantExpression(args[i], context); @@ -44,7 +44,7 @@ static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args } else /// Parsing arguments as Tuples { - for (size_t i = 1; i < args.size(); ++i) + for (size_t i = start; i < args.size(); ++i) { const auto & [value_field, value_type_ptr] = evaluateConstantExpression(args[i], context); @@ -68,34 +68,59 @@ static void parseAndInsertValues(MutableColumns & res_columns, const ASTs & args } } -void TableFunctionValues::parseArguments(const ASTPtr & ast_function, ContextPtr /*context*/) +DataTypes TableFunctionValues::getTypesFromArgument(const ASTPtr & arg, ContextPtr context) +{ + const auto & [value_field, value_type_ptr] = evaluateConstantExpression(arg, context); + DataTypes types; + if (const DataTypeTuple * type_tuple = typeid_cast(value_type_ptr.get())) + return type_tuple->getElements(); + + return {value_type_ptr}; +} + +void TableFunctionValues::parseArguments(const ASTPtr & ast_function, ContextPtr context) { ASTs & args_func = ast_function->children; if (args_func.size() != 1) - throw Exception("Table function '" + getName() + "' must have arguments.", ErrorCodes::LOGICAL_ERROR); + throw Exception("Table function '" + getName() + "' must have arguments", ErrorCodes::LOGICAL_ERROR); ASTs & args = args_func.at(0)->children; - if (args.size() < 2) - throw Exception("Table function '" + getName() + "' requires 2 or more arguments: structure and values.", - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); + if (args.empty()) + throw Exception("Table function '" + getName() + "' requires at least 1 argument", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - /// Parsing first argument as table structure and creating a sample block - if (!args[0]->as()) + const auto & literal = args[0]->as(); + String value; + if (args.size() > 1 && literal && literal->value.tryGet(value) && tryParseColumnsListFromString(value, structure, context)) { - throw Exception(fmt::format( - "The first argument of table function '{}' must be a literal. " - "Got '{}' instead", getName(), args[0]->formatForErrorMessage()), - ErrorCodes::BAD_ARGUMENTS); + has_structure_in_arguments = true; + return; } - structure = args[0]->as().value.safeGet(); + has_structure_in_arguments = false; + DataTypes data_types = getTypesFromArgument(args[0], context); + for (size_t i = 1; i < args.size(); ++i) + { + auto arg_types = getTypesFromArgument(args[i], context); + if (data_types.size() != arg_types.size()) + throw Exception( + ErrorCodes::CANNOT_EXTRACT_TABLE_STRUCTURE, + "Cannot determine common structure for {} function arguments: the amount of columns is differ for different arguments", + getName()); + for (size_t j = 0; j != arg_types.size(); ++j) + data_types[j] = getLeastSupertype({data_types[j], arg_types[j]}); + } + + NamesAndTypesList names_and_types; + for (size_t i = 0; i != data_types.size(); ++i) + names_and_types.emplace_back("c" + std::to_string(i + 1), data_types[i]); + structure = ColumnsDescription(names_and_types); } -ColumnsDescription TableFunctionValues::getActualTableStructure(ContextPtr context) const +ColumnsDescription TableFunctionValues::getActualTableStructure(ContextPtr /*context*/) const { - return parseColumnsListFromString(structure, context); + return structure; } StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription /*cached_columns*/) const @@ -111,7 +136,7 @@ StoragePtr TableFunctionValues::executeImpl(const ASTPtr & ast_function, Context ASTs & args = ast_function->children.at(0)->children; /// Parsing other arguments as values and inserting them into columns - parseAndInsertValues(res_columns, args, sample_block, context); + parseAndInsertValues(res_columns, args, sample_block, has_structure_in_arguments ? 1 : 0, context); Block res_block = sample_block.cloneWithColumns(std::move(res_columns)); diff --git a/src/TableFunctions/TableFunctionValues.h b/src/TableFunctions/TableFunctionValues.h index 058f5f1d2ed..f01bcf6e20e 100644 --- a/src/TableFunctions/TableFunctionValues.h +++ b/src/TableFunctions/TableFunctionValues.h @@ -20,7 +20,10 @@ private: ColumnsDescription getActualTableStructure(ContextPtr context) const override; void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - String structure; + static DataTypes getTypesFromArgument(const ASTPtr & arg, ContextPtr context); + + ColumnsDescription structure; + bool has_structure_in_arguments; }; diff --git a/src/TableFunctions/parseColumnsListForTableFunction.cpp b/src/TableFunctions/parseColumnsListForTableFunction.cpp index 08e80ef425a..911f2ae80f4 100644 --- a/src/TableFunctions/parseColumnsListForTableFunction.cpp +++ b/src/TableFunctions/parseColumnsListForTableFunction.cpp @@ -28,4 +28,24 @@ ColumnsDescription parseColumnsListFromString(const std::string & structure, Con return InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false); } +bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescription & columns, ContextPtr context) +{ + ParserColumnDeclarationList parser; + const Settings & settings = context->getSettingsRef(); + + String error; + const char * start = structure.data(); + const char * end = structure.data() + structure.size(); + ASTPtr columns_list_raw = tryParseQuery(parser, start, end, error, false, "columns declaration list", false, settings.max_query_size, settings.max_parser_depth); + if (!columns_list_raw) + return false; + + auto * columns_list = dynamic_cast(columns_list_raw.get()); + if (!columns_list) + return false; + + columns = InterpreterCreateQuery::getColumnsDescription(*columns_list, context, false); + return true; +} + } diff --git a/src/TableFunctions/parseColumnsListForTableFunction.h b/src/TableFunctions/parseColumnsListForTableFunction.h index e0130a2618d..e82a32f3d23 100644 --- a/src/TableFunctions/parseColumnsListForTableFunction.h +++ b/src/TableFunctions/parseColumnsListForTableFunction.h @@ -12,4 +12,6 @@ class Context; /// Parses a common argument for table functions such as table structure given in string ColumnsDescription parseColumnsListFromString(const std::string & structure, ContextPtr context); +bool tryParseColumnsListFromString(const std::string & structure, ColumnsDescription & columns, ContextPtr context); + } diff --git a/src/TableFunctions/registerTableFunctions.cpp b/src/TableFunctions/registerTableFunctions.cpp index ea5c2c75f94..ed08972e74d 100644 --- a/src/TableFunctions/registerTableFunctions.cpp +++ b/src/TableFunctions/registerTableFunctions.cpp @@ -49,6 +49,8 @@ void registerTableFunctions() #endif registerTableFunctionDictionary(factory); + + registerTableFunctionFormat(factory); } } diff --git a/src/TableFunctions/registerTableFunctions.h b/src/TableFunctions/registerTableFunctions.h index 8ddd9b7c8ab..72ca185f656 100644 --- a/src/TableFunctions/registerTableFunctions.h +++ b/src/TableFunctions/registerTableFunctions.h @@ -48,6 +48,8 @@ void registerTableFunctionSQLite(TableFunctionFactory & factory); void registerTableFunctionDictionary(TableFunctionFactory & factory); +void registerTableFunctionFormat(TableFunctionFactory & factory); + void registerTableFunctions(); } diff --git a/src/configure_config.cmake b/src/configure_config.cmake index ce50ab87afc..519307ba28a 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -4,6 +4,9 @@ endif() if (TARGET ch_contrib::bzip2) set(USE_BZIP2 1) endif() +if (TARGET ch_contrib::minizip) + set(USE_MINIZIP 1) +endif() if (TARGET ch_contrib::snappy) set(USE_SNAPPY 1) endif() diff --git a/tests/ci/ast_fuzzer_check.py b/tests/ci/ast_fuzzer_check.py index 042e0e90459..319a6fc3fa5 100644 --- a/tests/ci/ast_fuzzer_check.py +++ b/tests/ci/ast_fuzzer_check.py @@ -98,6 +98,7 @@ if __name__ == "__main__": 'server.log': os.path.join(workspace_path, 'server.log'), 'fuzzer.log': os.path.join(workspace_path, 'fuzzer.log'), 'report.html': os.path.join(workspace_path, 'report.html'), + 'core.gz': os.path.join(workspace_path, 'core.gz'), } s3_helper = S3Helper('https://s3.amazonaws.com') diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 70fdf06d40b..000d3d9a000 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -270,6 +270,9 @@ CI_CONFIG = { "Stateless tests (release, DatabaseReplicated, actions)": { "required_build": "package_release", }, + "Stateless tests (release, s3 storage, actions)": { + "required_build": "package_release", + }, "Stress test (address, actions)": { "required_build": "package_asan", }, diff --git a/tests/ci/docker_images_check.py b/tests/ci/docker_images_check.py index a908f5fe11c..140ede3067f 100644 --- a/tests/ci/docker_images_check.py +++ b/tests/ci/docker_images_check.py @@ -3,10 +3,11 @@ import argparse import json import logging import os +import platform import shutil import subprocess import time -from typing import List, Optional, Set, Tuple, Union +from typing import Dict, List, Optional, Set, Tuple, Union from github import Github @@ -23,24 +24,32 @@ NAME = "Push to Dockerhub (actions)" TEMP_PATH = os.path.join(RUNNER_TEMP, "docker_images_check") +ImagesDict = Dict[str, dict] + class DockerImage: def __init__( self, path: str, repo: str, + only_amd64: bool, parent: Optional["DockerImage"] = None, gh_repo_path: str = GITHUB_WORKSPACE, ): self.path = path self.full_path = os.path.join(gh_repo_path, path) self.repo = repo + self.only_amd64 = only_amd64 self.parent = parent self.built = False def __eq__(self, other) -> bool: # type: ignore """Is used to check if DockerImage is in a set or not""" - return self.path == other.path and self.repo == self.repo + return ( + self.path == other.path + and self.repo == self.repo + and self.only_amd64 == other.only_amd64 + ) def __lt__(self, other) -> bool: if not isinstance(other, DockerImage): @@ -65,9 +74,8 @@ class DockerImage: return f"DockerImage(path={self.path},repo={self.repo},parent={self.parent})" -def get_changed_docker_images( - pr_info: PRInfo, repo_path: str, image_file_path: str -) -> Set[DockerImage]: +def get_images_dict(repo_path: str, image_file_path: str) -> ImagesDict: + """Return images suppose to build on the current architecture host""" images_dict = {} path_to_images_file = os.path.join(repo_path, image_file_path) if os.path.exists(path_to_images_file): @@ -78,6 +86,13 @@ def get_changed_docker_images( "Image file %s doesnt exists in repo %s", image_file_path, repo_path ) + return images_dict + + +def get_changed_docker_images( + pr_info: PRInfo, images_dict: ImagesDict +) -> Set[DockerImage]: + if not images_dict: return set() @@ -96,6 +111,7 @@ def get_changed_docker_images( for f in files_changed: if f.startswith(dockerfile_dir): name = image_description["name"] + only_amd64 = image_description.get("only_amd64", False) logging.info( "Found changed file '%s' which affects " "docker image '%s' with path '%s'", @@ -103,7 +119,7 @@ def get_changed_docker_images( name, dockerfile_dir, ) - changed_images.append(DockerImage(dockerfile_dir, name)) + changed_images.append(DockerImage(dockerfile_dir, name, only_amd64)) break # The order is important: dependents should go later than bases, so that @@ -118,9 +134,9 @@ def get_changed_docker_images( dependent, image, ) - changed_images.append( - DockerImage(dependent, images_dict[dependent]["name"], image) - ) + name = images_dict[dependent]["name"] + only_amd64 = images_dict[dependent].get("only_amd64", False) + changed_images.append(DockerImage(dependent, name, only_amd64, image)) index += 1 if index > 5 * len(images_dict): # Sanity check to prevent infinite loop. @@ -161,12 +177,43 @@ def gen_versions( return versions, result_version +def build_and_push_dummy_image( + image: DockerImage, + version_string: str, + push: bool, +) -> Tuple[bool, str]: + dummy_source = "ubuntu:20.04" + logging.info("Building docker image %s as %s", image.repo, dummy_source) + build_log = os.path.join( + TEMP_PATH, f"build_and_push_log_{image.repo.replace('/', '_')}_{version_string}" + ) + with open(build_log, "wb") as bl: + cmd = ( + f"docker pull {dummy_source}; " + f"docker tag {dummy_source} {image.repo}:{version_string}; " + ) + if push: + cmd += f"docker push {image.repo}:{version_string}" + + logging.info("Docker command to run: %s", cmd) + with subprocess.Popen(cmd, shell=True, stderr=bl, stdout=bl) as proc: + retcode = proc.wait() + + if retcode != 0: + return False, build_log + + logging.info("Processing of %s successfully finished", image.repo) + return True, build_log + + def build_and_push_one_image( image: DockerImage, version_string: str, push: bool, child: bool, ) -> Tuple[bool, str]: + if image.only_amd64 and platform.machine() not in ["amd64", "x86_64"]: + return build_and_push_dummy_image(image, version_string, push) logging.info( "Building docker image %s with version %s from path %s", image.repo, @@ -290,10 +337,15 @@ def parse_args() -> argparse.Namespace: default="clickhouse", help="docker hub repository prefix", ) + parser.add_argument( + "--all", + action="store_true", + help="rebuild all images", + ) parser.add_argument( "--image-path", type=str, - action="append", + nargs="*", help="list of image paths to build instead of using pr_info + diff URL, " "e.g. 'docker/packager/binary'", ) @@ -336,15 +388,18 @@ def main(): shutil.rmtree(TEMP_PATH) os.makedirs(TEMP_PATH) - if args.image_path: + images_dict = get_images_dict(GITHUB_WORKSPACE, "docker/images.json") + + if args.all: + pr_info = PRInfo() + pr_info.changed_files = set(images_dict.keys()) + elif args.image_path: pr_info = PRInfo() pr_info.changed_files = set(i for i in args.image_path) else: pr_info = PRInfo(need_changed_files=True) - changed_images = get_changed_docker_images( - pr_info, GITHUB_WORKSPACE, "docker/images.json" - ) + changed_images = get_changed_docker_images(pr_info, images_dict) logging.info("Has changed images %s", ", ".join([im.path for im in changed_images])) image_versions, result_version = gen_versions(pr_info, args.suffix) diff --git a/tests/ci/docker_manifests_merge.py b/tests/ci/docker_manifests_merge.py index c6814b911ff..82d012bfe1a 100644 --- a/tests/ci/docker_manifests_merge.py +++ b/tests/ci/docker_manifests_merge.py @@ -57,7 +57,7 @@ def parse_args() -> argparse.Namespace: args = parser.parse_args() if len(args.suffixes) < 2: - raise parser.error("more than two --suffix should be given") + parser.error("more than two --suffix should be given") return args @@ -81,6 +81,7 @@ def strip_suffix(suffix: str, images: Images) -> Images: def check_sources(to_merge: Dict[str, Images]) -> Images: + """get a dict {arch1: Images, arch2: Images}""" result = {} # type: Images first_suffix = "" for suffix, images in to_merge.items(): diff --git a/tests/ci/docker_test.py b/tests/ci/docker_test.py index 4392641b215..27bfe07db53 100644 --- a/tests/ci/docker_test.py +++ b/tests/ci/docker_test.py @@ -23,54 +23,69 @@ class TestDockerImageCheck(unittest.TestCase): "docker/docs/builder", } images = sorted( - list(di.get_changed_docker_images(pr_info, "/", self.docker_images_path)) + list( + di.get_changed_docker_images( + pr_info, di.get_images_dict("/", self.docker_images_path) + ) + ) ) self.maxDiff = None expected = sorted( [ - di.DockerImage("docker/test/base", "clickhouse/test-base"), - di.DockerImage("docker/docs/builder", "clickhouse/docs-builder"), + di.DockerImage("docker/test/base", "clickhouse/test-base", False), + di.DockerImage("docker/docs/builder", "clickhouse/docs-builder", True), di.DockerImage( "docker/test/stateless", "clickhouse/stateless-test", + False, "clickhouse/test-base", ), di.DockerImage( "docker/test/integration/base", "clickhouse/integration-test", + False, "clickhouse/test-base", ), di.DockerImage( - "docker/test/fuzzer", "clickhouse/fuzzer", "clickhouse/test-base" + "docker/test/fuzzer", + "clickhouse/fuzzer", + False, + "clickhouse/test-base", ), di.DockerImage( "docker/test/keeper-jepsen", "clickhouse/keeper-jepsen-test", + False, "clickhouse/test-base", ), di.DockerImage( "docker/docs/check", "clickhouse/docs-check", + False, "clickhouse/docs-builder", ), di.DockerImage( "docker/docs/release", "clickhouse/docs-release", + False, "clickhouse/docs-builder", ), di.DockerImage( "docker/test/stateful", "clickhouse/stateful-test", + False, "clickhouse/stateless-test", ), di.DockerImage( "docker/test/unit", "clickhouse/unit-test", + False, "clickhouse/stateless-test", ), di.DockerImage( "docker/test/stress", "clickhouse/stress-test", + False, "clickhouse/stateful-test", ), ] @@ -92,13 +107,15 @@ class TestDockerImageCheck(unittest.TestCase): @patch("builtins.open") @patch("subprocess.Popen") - def test_build_and_push_one_image(self, mock_popen, mock_open): + @patch("platform.machine") + def test_build_and_push_one_image(self, mock_machine, mock_popen, mock_open): mock_popen.return_value.__enter__.return_value.wait.return_value = 0 - image = di.DockerImage("path", "name", gh_repo_path="") + image = di.DockerImage("path", "name", False, gh_repo_path="") result, _ = di.build_and_push_one_image(image, "version", True, True) mock_open.assert_called_once() mock_popen.assert_called_once() + mock_machine.assert_not_called() self.assertIn( "docker buildx build --builder default --build-arg FROM_TAG=version " "--build-arg BUILDKIT_INLINE_CACHE=1 --tag name:version --cache-from " @@ -106,11 +123,15 @@ class TestDockerImageCheck(unittest.TestCase): mock_popen.call_args.args, ) self.assertTrue(result) + mock_open.reset_mock() + mock_popen.reset_mock() + mock_machine.reset_mock() - mock_open.reset() - mock_popen.reset() mock_popen.return_value.__enter__.return_value.wait.return_value = 0 result, _ = di.build_and_push_one_image(image, "version2", False, True) + mock_open.assert_called_once() + mock_popen.assert_called_once() + mock_machine.assert_not_called() self.assertIn( "docker buildx build --builder default --build-arg FROM_TAG=version2 " "--build-arg BUILDKIT_INLINE_CACHE=1 --tag name:version2 --cache-from " @@ -119,8 +140,14 @@ class TestDockerImageCheck(unittest.TestCase): ) self.assertTrue(result) + mock_open.reset_mock() + mock_popen.reset_mock() + mock_machine.reset_mock() mock_popen.return_value.__enter__.return_value.wait.return_value = 1 result, _ = di.build_and_push_one_image(image, "version2", False, False) + mock_open.assert_called_once() + mock_popen.assert_called_once() + mock_machine.assert_not_called() self.assertIn( "docker buildx build --builder default " "--build-arg BUILDKIT_INLINE_CACHE=1 --tag name:version2 --cache-from " @@ -129,13 +156,37 @@ class TestDockerImageCheck(unittest.TestCase): ) self.assertFalse(result) + mock_open.reset_mock() + mock_popen.reset_mock() + mock_machine.reset_mock() + only_amd64_image = di.DockerImage("path", "name", True) + mock_popen.return_value.__enter__.return_value.wait.return_value = 0 + + result, _ = di.build_and_push_one_image(only_amd64_image, "version", True, True) + mock_open.assert_called_once() + mock_popen.assert_called_once() + mock_machine.assert_called_once() + self.assertIn( + "docker pull ubuntu:20.04; docker tag ubuntu:20.04 name:version; " + "docker push name:version", + mock_popen.call_args.args, + ) + self.assertTrue(result) + result, _ = di.build_and_push_one_image( + only_amd64_image, "version", False, True + ) + self.assertIn( + "docker pull ubuntu:20.04; docker tag ubuntu:20.04 name:version; ", + mock_popen.call_args.args, + ) + @patch("docker_images_check.build_and_push_one_image") def test_process_image_with_parents(self, mock_build): mock_build.side_effect = lambda w, x, y, z: (True, f"{w.repo}_{x}.log") - im1 = di.DockerImage("path1", "repo1") - im2 = di.DockerImage("path2", "repo2", im1) - im3 = di.DockerImage("path3", "repo3", im2) - im4 = di.DockerImage("path4", "repo4", im1) + im1 = di.DockerImage("path1", "repo1", False) + im2 = di.DockerImage("path2", "repo2", False, im1) + im3 = di.DockerImage("path3", "repo3", False, im2) + im4 = di.DockerImage("path4", "repo4", False, im1) # We use list to have determined order of image builgings images = [im4, im1, im3, im2, im1] results = [ diff --git a/tests/ci/functional_test_check.py b/tests/ci/functional_test_check.py index 7220b86a482..da08ff28a0f 100644 --- a/tests/ci/functional_test_check.py +++ b/tests/ci/functional_test_check.py @@ -30,6 +30,10 @@ def get_additional_envs(check_name, run_by_hash_num, run_by_hash_total): if 'wide parts enabled' in check_name: result.append("USE_POLYMORPHIC_PARTS=1") + #temporary + if 's3 storage' in check_name: + result.append("USE_S3_STORAGE_FOR_MERGE_TREE=1") + if run_by_hash_total != 0: result.append(f"RUN_BY_HASH_NUM={run_by_hash_num}") result.append(f"RUN_BY_HASH_TOTAL={run_by_hash_total}") @@ -44,7 +48,7 @@ def get_image_name(check_name): else: raise Exception(f"Cannot deduce image name based on check name {check_name}") -def get_run_command(builds_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): +def get_run_command(builds_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, image, flaky_check, tests_to_run): additional_options = ['--hung-check'] additional_options.append('--print-time') @@ -63,6 +67,7 @@ def get_run_command(builds_path, result_path, server_log_path, kill_timeout, add env_str = ' '.join(envs) return f"docker run --volume={builds_path}:/package_folder " \ + f"--volume={repo_tests_path}:/usr/share/clickhouse-test " \ f"--volume={result_path}:/test_output --volume={server_log_path}:/var/log/clickhouse-server " \ f"--cap-add=SYS_PTRACE {env_str} {additional_options_str} {image}" @@ -167,6 +172,8 @@ if __name__ == "__main__": image_name = get_image_name(check_name) docker_image = get_image_with_version(reports_path, image_name) + repo_tests_path = os.path.join(repo_path, "tests") + packages_path = os.path.join(temp_path, "packages") if not os.path.exists(packages_path): os.makedirs(packages_path) @@ -184,7 +191,7 @@ if __name__ == "__main__": run_log_path = os.path.join(result_path, "runlog.log") additional_envs = get_additional_envs(check_name, run_by_hash_num, run_by_hash_total) - run_command = get_run_command(packages_path, result_path, server_log_path, kill_timeout, additional_envs, docker_image, flaky_check, tests_to_run) + run_command = get_run_command(packages_path, repo_tests_path, result_path, server_log_path, kill_timeout, additional_envs, docker_image, flaky_check, tests_to_run) logging.info("Going to run func tests: %s", run_command) with TeePopen(run_command, run_log_path) as process: diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index e87528dd528..786521db418 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -23,6 +23,8 @@ from rerun_helper import RerunHelper from tee_popen import TeePopen +# When update, update +# integration/ci-runner.py:ClickhouseIntegrationTestsRunner.get_images_names too IMAGES = [ "clickhouse/integration-tests-runner", "clickhouse/mysql-golang-client", @@ -32,6 +34,7 @@ IMAGES = [ "clickhouse/postgresql-java-client", "clickhouse/integration-test", "clickhouse/kerberos-kdc", + "clickhouse/kerberized-hadoop", "clickhouse/integration-helper", "clickhouse/dotnet-client", ] diff --git a/tests/ci/run_check.py b/tests/ci/run_check.py index 452824d58be..a2403e61ac1 100644 --- a/tests/ci/run_check.py +++ b/tests/ci/run_check.py @@ -2,8 +2,9 @@ import sys import logging import re -from github import Github +from typing import Tuple +from github import Github from env_helper import GITHUB_RUN_ID, GITHUB_REPOSITORY, GITHUB_SERVER_URL from pr_info import PRInfo from get_robot_token import get_best_robot_token @@ -17,8 +18,10 @@ TRUSTED_ORG_IDS = { 54801242, # clickhouse } -OK_TEST_LABEL = set(["can be tested", "release", "pr-documentation", "pr-doc-fix"]) +OK_SKIP_LABELS = {"release", "pr-backport", "pr-cherrypick"} +CAN_BE_TESTED_LABEL = "can be tested" DO_NOT_TEST_LABEL = "do not test" +FORCE_TESTS_LABEL = "force tests" # Individual trusted contirbutors who are not in any trusted organization. # Can be changed in runtime: we will append users that we learned to be in @@ -100,29 +103,29 @@ def pr_is_by_trusted_user(pr_user_login, pr_user_orgs): # Returns whether we should look into individual checks for this PR. If not, it # can be skipped entirely. -def should_run_checks_for_pr(pr_info): +# Returns can_run, description, labels_state +def should_run_checks_for_pr(pr_info: PRInfo) -> Tuple[bool, str, str]: # Consider the labels and whether the user is trusted. print("Got labels", pr_info.labels) - force_labels = set(["force tests"]).intersection(pr_info.labels) - if force_labels: - return True, "Labeled '{}'".format(", ".join(force_labels)) + if FORCE_TESTS_LABEL in pr_info.labels: + return True, f"Labeled '{FORCE_TESTS_LABEL}'", "pending" - if "do not test" in pr_info.labels: - return False, "Labeled 'do not test'" + if DO_NOT_TEST_LABEL in pr_info.labels: + return False, f"Labeled '{DO_NOT_TEST_LABEL}'", "success" - if "can be tested" not in pr_info.labels and not pr_is_by_trusted_user( + if CAN_BE_TESTED_LABEL not in pr_info.labels and not pr_is_by_trusted_user( pr_info.user_login, pr_info.user_orgs ): - return False, "Needs 'can be tested' label" + return False, "Needs 'can be tested' label", "failure" - if ( - "release" in pr_info.labels - or "pr-backport" in pr_info.labels - or "pr-cherrypick" in pr_info.labels - ): - return False, "Don't try new checks for release/backports/cherry-picks" + if OK_SKIP_LABELS.intersection(pr_info.labels): + return ( + False, + "Don't try new checks for release/backports/cherry-picks", + "success", + ) - return True, "No special conditions apply" + return True, "No special conditions apply", "pending" def check_pr_description(pr_info): @@ -205,7 +208,7 @@ if __name__ == "__main__": logging.basicConfig(level=logging.INFO) pr_info = PRInfo(need_orgs=True, pr_event_from_api=True) - can_run, description = should_run_checks_for_pr(pr_info) + can_run, description, labels_state = should_run_checks_for_pr(pr_info) gh = Github(get_best_robot_token()) commit = get_commit(gh, pr_info.sha) @@ -231,7 +234,7 @@ if __name__ == "__main__": if not can_run: print("::notice ::Cannot run") commit.create_status( - context=NAME, description=description, state="failure", target_url=url + context=NAME, description=description, state=labels_state, target_url=url ) sys.exit(1) else: diff --git a/tests/ci/style_check.py b/tests/ci/style_check.py index 8490b33dd22..1b3037217c8 100644 --- a/tests/ci/style_check.py +++ b/tests/ci/style_check.py @@ -86,12 +86,18 @@ if __name__ == "__main__": docker_image = get_image_with_version(temp_path, "clickhouse/style-test") s3_helper = S3Helper("https://s3.amazonaws.com") - subprocess.check_output( + cmd = ( f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE " f"--volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output " - f"{docker_image}", + f"{docker_image}" + ) + + logging.info("Is going to run the command: %s", cmd) + subprocess.check_call( + cmd, shell=True, ) + state, description, test_results, additional_files = process_result(temp_path) ch_helper = ClickHouseHelper() mark_flaky_tests(ch_helper, NAME, test_results) diff --git a/tests/ci/tests/docker_images.json b/tests/ci/tests/docker_images.json index 354bdaa8728..ca5c516bccb 100644 --- a/tests/ci/tests/docker_images.json +++ b/tests/ci/tests/docker_images.json @@ -150,6 +150,7 @@ }, "docker/docs/builder": { "name": "clickhouse/docs-builder", + "only_amd64": true, "dependent": [ "docker/docs/check", "docker/docs/release" diff --git a/tests/ci/worker/dockerhub_proxy_template.sh b/tests/ci/worker/dockerhub_proxy_template.sh index 5ee63a05125..add15158d94 100644 --- a/tests/ci/worker/dockerhub_proxy_template.sh +++ b/tests/ci/worker/dockerhub_proxy_template.sh @@ -5,4 +5,7 @@ mkdir /home/ubuntu/registrystorage sed -i 's/preserve_hostname: false/preserve_hostname: true/g' /etc/cloud/cloud.cfg -docker run -d --network=host -p 5000:5000 -v /home/ubuntu/registrystorage:/var/lib/registry -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 -e REGISTRY_STORAGE_DELETE_ENABLED=true -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io --restart=always --name registry registry:2 +REGISTRY_PROXY_USERNAME=robotclickhouse +REGISTRY_PROXY_PASSWORD=$(aws ssm get-parameter --name dockerhub_robot_password --with-decryption | jq '.Parameter.Value' -r) + +docker run -d --network=host -p 5000:5000 -v /home/ubuntu/registrystorage:/var/lib/registry -e REGISTRY_HTTP_ADDR=0.0.0.0:5000 -e REGISTRY_STORAGE_DELETE_ENABLED=true -e REGISTRY_PROXY_REMOTEURL=https://registry-1.docker.io -e REGISTRY_PROXY_PASSWORD="$REGISTRY_PROXY_PASSWORD" -e REGISTRY_PROXY_USERNAME="$REGISTRY_PROXY_USERNAME" --restart=always --name registry registry:2 diff --git a/tests/clickhouse-test b/tests/clickhouse-test index 793c700ab3f..0969a1c1697 100755 --- a/tests/clickhouse-test +++ b/tests/clickhouse-test @@ -329,6 +329,7 @@ class FailureReason(enum.Enum): FAST_ONLY = "running fast tests only" NO_LONG = "not running long tests" REPLICATED_DB = "replicated-database" + S3_STORAGE = "s3-storage" BUILD = "not running for current build" # UNKNOWN reasons @@ -519,6 +520,10 @@ class TestCase: elif tags and ('no-replicated-database' in tags) and args.replicated_database: return FailureReason.REPLICATED_DB + elif tags and ('no-s3-storage' in tags) and args.s3_storage: + return FailureReason.S3_STORAGE + + elif tags: for build_flag in args.build_flags: if 'no-' + build_flag in tags: @@ -1431,6 +1436,7 @@ if __name__ == '__main__': parser.add_argument('--client-option', nargs='+', help='Specify additional client argument') parser.add_argument('--print-time', action='store_true', dest='print_time', help='Print test time') parser.add_argument('--check-zookeeper-session', action='store_true', help='Check ZooKeeper session uptime to determine if failed test should be retried') + parser.add_argument('--s3-storage', action='store_true', default=False, help='Run tests over s3 storage') parser.add_argument('--run-by-hash-num', type=int, help='Run tests matching crc32(test_name) % run_by_hash_total == run_by_hash_num') parser.add_argument('--run-by-hash-total', type=int, help='Total test groups for crc32(test_name) % run_by_hash_total == run_by_hash_num') diff --git a/tests/config/config.d/s3_storage_policy_by_default.xml b/tests/config/config.d/s3_storage_policy_by_default.xml new file mode 100644 index 00000000000..6ce997a2c16 --- /dev/null +++ b/tests/config/config.d/s3_storage_policy_by_default.xml @@ -0,0 +1,24 @@ + + + + + s3 + http://localhost:11111/test/test/ + clickhouse + clickhouse + + + + + +

+ s3 +
+ + + + + + s3 + + diff --git a/tests/config/install.sh b/tests/config/install.sh index 731a4317f44..320a3d0f799 100755 --- a/tests/config/install.sh +++ b/tests/config/install.sh @@ -107,4 +107,8 @@ if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]] sudo chgrp clickhouse /var/lib/clickhouse2 fi +if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then + ln -sf $SRC_PATH/config.d/s3_storage_policy_by_default.xml $DEST_SERVER_PATH/config.d/ +fi + ln -sf $SRC_PATH/client_config.xml $DEST_CLIENT_PATH/config.xml diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 40cb2c6fdd7..8f228d91e9e 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -1,17 +1,16 @@ #!/usr/bin/env python3 -import logging -import subprocess -import os -import glob -import time -import shutil from collections import defaultdict -import random -import json import csv -# for crc32 -import zlib +import glob +import json +import logging +import os +import random +import shutil +import subprocess +import time +import zlib # for crc32 MAX_RETRY = 3 @@ -25,54 +24,62 @@ CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH = "usr/bin/clickhouse-library-bridge" TRIES_COUNT = 10 MAX_TIME_SECONDS = 3600 -MAX_TIME_IN_SANDBOX = 20 * 60 # 20 minutes -TASK_TIMEOUT = 8 * 60 * 60 # 8 hours +MAX_TIME_IN_SANDBOX = 20 * 60 # 20 minutes +TASK_TIMEOUT = 8 * 60 * 60 # 8 hours + def stringhash(s): - return zlib.crc32(s.encode('utf-8')) + return zlib.crc32(s.encode("utf-8")) + def get_tests_to_run(pr_info): result = set([]) - changed_files = pr_info['changed_files'] + changed_files = pr_info["changed_files"] if changed_files is None: return [] for fpath in changed_files: - if 'tests/integration/test_' in fpath: - logging.info('File %s changed and seems like integration test', fpath) - result.add(fpath.split('/')[2]) + if "tests/integration/test_" in fpath: + logging.info("File %s changed and seems like integration test", fpath) + result.add(fpath.split("/")[2]) return list(result) def filter_existing_tests(tests_to_run, repo_path): result = [] for relative_test_path in tests_to_run: - if os.path.exists(os.path.join(repo_path, 'tests/integration', relative_test_path)): + if os.path.exists( + os.path.join(repo_path, "tests/integration", relative_test_path) + ): result.append(relative_test_path) else: - logging.info("Skipping test %s, seems like it was removed", relative_test_path) + logging.info( + "Skipping test %s, seems like it was removed", relative_test_path + ) return result def _get_deselect_option(tests): - return ' '.join(['--deselect {}'.format(t) for t in tests]) + return " ".join([f"--deselect {t}" for t in tests]) + # https://stackoverflow.com/questions/312443/how-do-you-split-a-list-into-evenly-sized-chunks def chunks(lst, n): """Yield successive n-sized chunks from lst.""" for i in range(0, len(lst), n): - yield lst[i:i + n] + yield lst[i : i + n] + def get_counters(fname): counters = { - "ERROR": set([]), - "PASSED": set([]), - "FAILED": set([]), + "ERROR": set([]), + "PASSED": set([]), + "FAILED": set([]), "SKIPPED": set([]), } - with open(fname, 'r') as out: + with open(fname, "r") as out: for line in out: line = line.strip() # Example of log: @@ -81,10 +88,10 @@ def get_counters(fname): # [gw0] [ 7%] ERROR test_mysql_protocol/test.py::test_golang_client # # And only the line with test status should be matched - if not('.py::' in line and ' ' in line): + if not (".py::" in line and " " in line): continue - line_arr = line.strip().split(' ') + line_arr = line.strip().split(" ") if len(line_arr) < 2: logging.debug("Strange line %s", line) continue @@ -97,9 +104,9 @@ def get_counters(fname): if state in counters: counters[state].add(test_name) else: - # will skip lines line: - # 30.76s call test_host_ip_change/test.py::test_ip_change_drop_dns_cache - # 5.71s teardown test_host_ip_change/test.py::test_user_access_ip_change[node1] + # will skip lines like: + # 30.76s call test_host_ip_change/test.py::test_ip_drop_cache + # 5.71s teardown test_host_ip_change/test.py::test_ip_change[node1] # and similar logging.debug("Strange state in line %s", line) @@ -109,13 +116,13 @@ def get_counters(fname): def parse_test_times(fname): read = False description_output = [] - with open(fname, 'r') as out: + with open(fname, "r") as out: for line in out: - if read and '==' in line: + if read and "==" in line: break if read and line.strip(): description_output.append(line.strip()) - if 'slowest durations' in line: + if "slowest durations" in line: read = True return description_output @@ -123,10 +130,10 @@ def parse_test_times(fname): def get_test_times(output): result = defaultdict(float) for line in output: - if '.py' in line: - line_arr = line.strip().split(' ') + if ".py" in line: + line_arr = line.strip().split(" ") test_time = line_arr[0] - test_name = ' '.join([elem for elem in line_arr[2:] if elem]) + test_name = " ".join([elem for elem in line_arr[2:] if elem]) if test_name not in result: result[test_name] = 0.0 result[test_name] += float(test_time[:-1]) @@ -134,21 +141,28 @@ def get_test_times(output): def clear_ip_tables_and_restart_daemons(): - logging.info("Dump iptables after run %s", subprocess.check_output("sudo iptables -L", shell=True)) + logging.info( + "Dump iptables after run %s", + subprocess.check_output("sudo iptables -L", shell=True), + ) try: logging.info("Killing all alive docker containers") - subprocess.check_output("timeout -s 9 10m docker kill $(docker ps -q)", shell=True) + subprocess.check_output( + "timeout -s 9 10m docker kill $(docker ps -q)", shell=True + ) except subprocess.CalledProcessError as err: logging.info("docker kill excepted: " + str(err)) try: logging.info("Removing all docker containers") - subprocess.check_output("timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True) + subprocess.check_output( + "timeout -s 9 10m docker rm $(docker ps -a -q) --force", shell=True + ) except subprocess.CalledProcessError as err: logging.info("docker rm excepted: " + str(err)) # don't restart docker if it's disabled - if os.environ.get("CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER", '1') == '1': + if os.environ.get("CLICKHOUSE_TESTS_RUNNER_RESTART_DOCKER", "1") == "1": try: logging.info("Stopping docker daemon") subprocess.check_output("service docker stop", shell=True) @@ -177,27 +191,33 @@ def clear_ip_tables_and_restart_daemons(): # when rules will be empty, it will raise exception subprocess.check_output("sudo iptables -D DOCKER-USER 1", shell=True) except subprocess.CalledProcessError as err: - logging.info("All iptables rules cleared, " + str(iptables_iter) + "iterations, last error: " + str(err)) + logging.info( + "All iptables rules cleared, " + + str(iptables_iter) + + "iterations, last error: " + + str(err) + ) class ClickhouseIntegrationTestsRunner: - def __init__(self, result_path, params): self.result_path = result_path self.params = params - self.image_versions = self.params['docker_images_with_versions'] - self.shuffle_groups = self.params['shuffle_test_groups'] - self.flaky_check = 'flaky check' in self.params['context_name'] + self.image_versions = self.params["docker_images_with_versions"] + self.shuffle_groups = self.params["shuffle_test_groups"] + self.flaky_check = "flaky check" in self.params["context_name"] # if use_tmpfs is not set we assume it to be true, otherwise check - self.use_tmpfs = 'use_tmpfs' not in self.params or self.params['use_tmpfs'] - self.disable_net_host = 'disable_net_host' in self.params and self.params['disable_net_host'] + self.use_tmpfs = "use_tmpfs" not in self.params or self.params["use_tmpfs"] + self.disable_net_host = ( + "disable_net_host" in self.params and self.params["disable_net_host"] + ) self.start_time = time.time() self.soft_deadline_time = self.start_time + (TASK_TIMEOUT - MAX_TIME_IN_SANDBOX) - if 'run_by_hash_total' in self.params: - self.run_by_hash_total = self.params['run_by_hash_total'] - self.run_by_hash_num = self.params['run_by_hash_num'] + if "run_by_hash_total" in self.params: + self.run_by_hash_total = self.params["run_by_hash_total"] + self.run_by_hash_num = self.params["run_by_hash_num"] else: self.run_by_hash_total = 0 self.run_by_hash_num = 0 @@ -206,7 +226,7 @@ class ClickhouseIntegrationTestsRunner: return self.result_path def base_path(self): - return os.path.join(str(self.result_path), '../') + return os.path.join(str(self.result_path), "../") def should_skip_tests(self): return [] @@ -214,8 +234,10 @@ class ClickhouseIntegrationTestsRunner: def get_image_with_version(self, name): if name in self.image_versions: return name + ":" + self.image_versions[name] - logging.warn("Cannot find image %s in params list %s", name, self.image_versions) - if ':' not in name: + logging.warn( + "Cannot find image %s in params list %s", name, self.image_versions + ) + if ":" not in name: return name + ":latest" return name @@ -223,31 +245,44 @@ class ClickhouseIntegrationTestsRunner: name = self.get_images_names()[0] if name in self.image_versions: return self.image_versions[name] - logging.warn("Cannot find image %s in params list %s", name, self.image_versions) - return 'latest' + logging.warn( + "Cannot find image %s in params list %s", name, self.image_versions + ) + return "latest" def shuffle_test_groups(self): return self.shuffle_groups != 0 @staticmethod def get_images_names(): - return ["clickhouse/integration-tests-runner", "clickhouse/mysql-golang-client", - "clickhouse/mysql-java-client", "clickhouse/mysql-js-client", - "clickhouse/mysql-php-client", "clickhouse/postgresql-java-client", - "clickhouse/integration-test", "clickhouse/kerberos-kdc", - "clickhouse/dotnet-client", - "clickhouse/integration-helper", ] - + return [ + "clickhouse/dotnet-client", + "clickhouse/integration-helper", + "clickhouse/integration-test", + "clickhouse/integration-tests-runner", + "clickhouse/kerberized-hadoop", + "clickhouse/kerberos-kdc", + "clickhouse/mysql-golang-client", + "clickhouse/mysql-java-client", + "clickhouse/mysql-js-client", + "clickhouse/mysql-php-client", + "clickhouse/postgresql-java-client", + ] def _can_run_with(self, path, opt): - with open(path, 'r') as script: + with open(path, "r") as script: for line in script: if opt in line: return True return False def _install_clickhouse(self, debs_path): - for package in ('clickhouse-common-static_', 'clickhouse-server_', 'clickhouse-client', 'clickhouse-common-static-dbg_'): # order matters + for package in ( + "clickhouse-common-static_", + "clickhouse-server_", + "clickhouse-client", + "clickhouse-common-static-dbg_", + ): # order matters logging.info("Installing package %s", package) for f in os.listdir(debs_path): if package in f: @@ -255,10 +290,12 @@ class ClickhouseIntegrationTestsRunner: logging.info("Package found in %s", full_path) log_name = "install_" + f + ".log" log_path = os.path.join(str(self.path()), log_name) - with open(log_path, 'w') as log: + with open(log_path, "w") as log: cmd = "dpkg -x {} .".format(full_path) logging.info("Executing installation cmd %s", cmd) - retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait() + retcode = subprocess.Popen( + cmd, shell=True, stderr=log, stdout=log + ).wait() if retcode == 0: logging.info("Installation of %s successfull", full_path) else: @@ -267,18 +304,35 @@ class ClickhouseIntegrationTestsRunner: else: raise Exception("Package with {} not found".format(package)) logging.info("Unstripping binary") - # logging.info("Unstring %s", subprocess.check_output("eu-unstrip /usr/bin/clickhouse {}".format(CLICKHOUSE_BINARY_PATH), shell=True)) + # logging.info( + # "Unstring %s", + # subprocess.check_output( + # "eu-unstrip /usr/bin/clickhouse {}".format(CLICKHOUSE_BINARY_PATH), + # shell=True, + # ), + # ) logging.info("All packages installed") os.chmod(CLICKHOUSE_BINARY_PATH, 0o777) os.chmod(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, 0o777) os.chmod(CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, 0o777) - shutil.copy(CLICKHOUSE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH")) - shutil.copy(CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH")) - shutil.copy(CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH")) + shutil.copy( + CLICKHOUSE_BINARY_PATH, os.getenv("CLICKHOUSE_TESTS_SERVER_BIN_PATH") + ) + shutil.copy( + CLICKHOUSE_ODBC_BRIDGE_BINARY_PATH, + os.getenv("CLICKHOUSE_TESTS_ODBC_BRIDGE_BIN_PATH"), + ) + shutil.copy( + CLICKHOUSE_LIBRARY_BRIDGE_BINARY_PATH, + os.getenv("CLICKHOUSE_TESTS_LIBRARY_BRIDGE_BIN_PATH"), + ) def _compress_logs(self, dir, relpaths, result_path): - subprocess.check_call("tar czf {} -C {} {}".format(result_path, dir, ' '.join(relpaths)), shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + subprocess.check_call( # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + "tar czf {} -C {} {}".format(result_path, dir, " ".join(relpaths)), + shell=True, + ) def _get_runner_opts(self): result = [] @@ -292,22 +346,40 @@ class ClickhouseIntegrationTestsRunner: image_cmd = self._get_runner_image_cmd(repo_path) out_file = "all_tests.txt" out_file_full = "all_tests_full.txt" - cmd = "cd {repo_path}/tests/integration && " \ - "timeout -s 9 1h ./runner {runner_opts} {image_cmd} ' --setup-plan' " \ - "| tee {out_file_full} | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' | sed 's/ *$//g' " \ + cmd = ( + "cd {repo_path}/tests/integration && " + "timeout -s 9 1h ./runner {runner_opts} {image_cmd} ' --setup-plan' " + "| tee {out_file_full} | grep '::' | sed 's/ (fixtures used:.*//g' | sed 's/^ *//g' | sed 's/ *$//g' " "| grep -v 'SKIPPED' | sort -u > {out_file}".format( - repo_path=repo_path, runner_opts=self._get_runner_opts(), image_cmd=image_cmd, out_file=out_file, out_file_full=out_file_full) + repo_path=repo_path, + runner_opts=self._get_runner_opts(), + image_cmd=image_cmd, + out_file=out_file, + out_file_full=out_file_full, + ) + ) logging.info("Getting all tests with cmd '%s'", cmd) - subprocess.check_call(cmd, shell=True) # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + subprocess.check_call( # STYLE_CHECK_ALLOW_SUBPROCESS_CHECK_CALL + cmd, shell=True + ) - all_tests_file_path = "{repo_path}/tests/integration/{out_file}".format(repo_path=repo_path, out_file=out_file) - if not os.path.isfile(all_tests_file_path) or os.path.getsize(all_tests_file_path) == 0: - all_tests_full_file_path = "{repo_path}/tests/integration/{out_file}".format(repo_path=repo_path, out_file=out_file_full) + all_tests_file_path = "{repo_path}/tests/integration/{out_file}".format( + repo_path=repo_path, out_file=out_file + ) + if ( + not os.path.isfile(all_tests_file_path) + or os.path.getsize(all_tests_file_path) == 0 + ): + all_tests_full_file_path = ( + "{repo_path}/tests/integration/{out_file}".format( + repo_path=repo_path, out_file=out_file_full + ) + ) if os.path.isfile(all_tests_full_file_path): # log runner output logging.info("runner output:") - with open(all_tests_full_file_path, 'r') as all_tests_full_file: + with open(all_tests_full_file_path, "r") as all_tests_full_file: for line in all_tests_full_file: line = line.rstrip() if line: @@ -315,7 +387,11 @@ class ClickhouseIntegrationTestsRunner: else: logging.info("runner output '%s' is empty", all_tests_full_file_path) - raise Exception("There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(all_tests_file_path)) + raise Exception( + "There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format( + all_tests_file_path + ) + ) all_tests = [] with open(all_tests_file_path, "r") as all_tests_file: @@ -324,9 +400,18 @@ class ClickhouseIntegrationTestsRunner: return list(sorted(all_tests)) def _get_parallel_tests_skip_list(self, repo_path): - skip_list_file_path = "{}/tests/integration/parallel_skip.json".format(repo_path) - if not os.path.isfile(skip_list_file_path) or os.path.getsize(skip_list_file_path) == 0: - raise Exception("There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format(skip_list_file_path)) + skip_list_file_path = "{}/tests/integration/parallel_skip.json".format( + repo_path + ) + if ( + not os.path.isfile(skip_list_file_path) + or os.path.getsize(skip_list_file_path) == 0 + ): + raise Exception( + "There is something wrong with getting all tests list: file '{}' is empty or does not exist.".format( + skip_list_file_path + ) + ) skip_list_tests = [] with open(skip_list_file_path, "r") as skip_list_file: @@ -336,7 +421,7 @@ class ClickhouseIntegrationTestsRunner: def group_test_by_file(self, tests): result = {} for test in tests: - test_file = test.split('::')[0] + test_file = test.split("::")[0] if test_file not in result: result[test_file] = [] result[test_file].append(test) @@ -344,7 +429,10 @@ class ClickhouseIntegrationTestsRunner: def _update_counters(self, main_counters, current_counters): for test in current_counters["PASSED"]: - if test not in main_counters["PASSED"] and test not in main_counters["FLAKY"]: + if ( + test not in main_counters["PASSED"] + and test not in main_counters["FLAKY"] + ): is_flaky = False if test in main_counters["FAILED"]: main_counters["FAILED"].remove(test) @@ -369,45 +457,63 @@ class ClickhouseIntegrationTestsRunner: main_counters[state].append(test) def _get_runner_image_cmd(self, repo_path): - image_cmd = '' - if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-image-version'): + image_cmd = "" + if self._can_run_with( + os.path.join(repo_path, "tests/integration", "runner"), + "--docker-image-version", + ): for img in self.get_images_names(): if img == "clickhouse/integration-tests-runner": runner_version = self.get_single_image_version() - logging.info("Can run with custom docker image version %s", runner_version) - image_cmd += ' --docker-image-version={} '.format(runner_version) + logging.info( + "Can run with custom docker image version %s", runner_version + ) + image_cmd += " --docker-image-version={} ".format(runner_version) else: - if self._can_run_with(os.path.join(repo_path, "tests/integration", "runner"), '--docker-compose-images-tags'): - image_cmd += '--docker-compose-images-tags={} '.format(self.get_image_with_version(img)) + if self._can_run_with( + os.path.join(repo_path, "tests/integration", "runner"), + "--docker-compose-images-tags", + ): + image_cmd += "--docker-compose-images-tags={} ".format( + self.get_image_with_version(img) + ) else: - image_cmd = '' + image_cmd = "" logging.info("Cannot run with custom docker image version :(") return image_cmd def _find_test_data_dirs(self, repo_path, test_names): relpaths = {} for test_name in test_names: - if '/' in test_name: - test_dir = test_name[:test_name.find('/')] + if "/" in test_name: + test_dir = test_name[: test_name.find("/")] else: test_dir = test_name if os.path.isdir(os.path.join(repo_path, "tests/integration", test_dir)): - for name in os.listdir(os.path.join(repo_path, "tests/integration", test_dir)): + for name in os.listdir( + os.path.join(repo_path, "tests/integration", test_dir) + ): relpath = os.path.join(os.path.join(test_dir, name)) - mtime = os.path.getmtime(os.path.join(repo_path, "tests/integration", relpath)) + mtime = os.path.getmtime( + os.path.join(repo_path, "tests/integration", relpath) + ) relpaths[relpath] = mtime return relpaths def _get_test_data_dirs_difference(self, new_snapshot, old_snapshot): res = set() for path in new_snapshot: - if (not path in old_snapshot) or (old_snapshot[path] != new_snapshot[path]): + if (path not in old_snapshot) or (old_snapshot[path] != new_snapshot[path]): res.add(path) return res - def try_run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers): + def try_run_test_group( + self, repo_path, test_group, tests_in_group, num_tries, num_workers + ): try: - return self.run_test_group(repo_path, test_group, tests_in_group, num_tries, num_workers) + return self.run_test_group( + repo_path, test_group, tests_in_group, num_tries, num_workers + ) except Exception as e: logging.info("Failed to run {}:\n{}".format(str(test_group), str(e))) counters = { @@ -423,7 +529,9 @@ class ClickhouseIntegrationTestsRunner: tests_times[test] = 0 return counters, tests_times, [] - def run_test_group(self, repo_path, test_group, tests_in_group, num_tries, num_workers): + def run_test_group( + self, repo_path, test_group, tests_in_group, num_tries, num_workers + ): counters = { "ERROR": [], "PASSED": [], @@ -441,7 +549,7 @@ class ClickhouseIntegrationTestsRunner: return counters, tests_times, [] image_cmd = self._get_runner_image_cmd(repo_path) - test_group_str = test_group.replace('/', '_').replace('.', '_') + test_group_str = test_group.replace("/", "_").replace(".", "_") log_paths = [] test_data_dirs = {} @@ -453,8 +561,8 @@ class ClickhouseIntegrationTestsRunner: test_names = set([]) for test_name in tests_in_group: if test_name not in counters["PASSED"]: - if '[' in test_name: - test_names.add(test_name[:test_name.find('[')]) + if "[" in test_name: + test_names.add(test_name[: test_name.find("[")]) else: test_names.add(test_name) @@ -464,47 +572,83 @@ class ClickhouseIntegrationTestsRunner: info_basename = test_group_str + "_" + str(i) + ".nfo" info_path = os.path.join(repo_path, "tests/integration", info_basename) - test_cmd = ' '.join([test for test in sorted(test_names)]) - parallel_cmd = " --parallel {} ".format(num_workers) if num_workers > 0 else "" + test_cmd = " ".join([test for test in sorted(test_names)]) + parallel_cmd = ( + " --parallel {} ".format(num_workers) if num_workers > 0 else "" + ) # -r -- show extra test summary: # -f -- (f)ailed # -E -- (E)rror # -p -- (p)assed # -s -- (s)kipped cmd = "cd {}/tests/integration && timeout -s 9 1h ./runner {} {} -t {} {} '-rfEps --run-id={} --color=no --durations=0 {}' | tee {}".format( - repo_path, self._get_runner_opts(), image_cmd, test_cmd, parallel_cmd, i, _get_deselect_option(self.should_skip_tests()), info_path) + repo_path, + self._get_runner_opts(), + image_cmd, + test_cmd, + parallel_cmd, + i, + _get_deselect_option(self.should_skip_tests()), + info_path, + ) log_basename = test_group_str + "_" + str(i) + ".log" log_path = os.path.join(repo_path, "tests/integration", log_basename) - with open(log_path, 'w') as log: + with open(log_path, "w") as log: logging.info("Executing cmd: %s", cmd) - retcode = subprocess.Popen(cmd, shell=True, stderr=log, stdout=log).wait() + retcode = subprocess.Popen( + cmd, shell=True, stderr=log, stdout=log + ).wait() if retcode == 0: logging.info("Run %s group successfully", test_group) else: logging.info("Some tests failed") extra_logs_names = [log_basename] - log_result_path = os.path.join(str(self.path()), 'integration_run_' + log_basename) + log_result_path = os.path.join( + str(self.path()), "integration_run_" + log_basename + ) shutil.copy(log_path, log_result_path) log_paths.append(log_result_path) - for pytest_log_path in glob.glob(os.path.join(repo_path, "tests/integration/pytest*.log")): - new_name = test_group_str + "_" + str(i) + "_" + os.path.basename(pytest_log_path) - os.rename(pytest_log_path, os.path.join(repo_path, "tests/integration", new_name)) + for pytest_log_path in glob.glob( + os.path.join(repo_path, "tests/integration/pytest*.log") + ): + new_name = ( + test_group_str + + "_" + + str(i) + + "_" + + os.path.basename(pytest_log_path) + ) + os.rename( + pytest_log_path, + os.path.join(repo_path, "tests/integration", new_name), + ) extra_logs_names.append(new_name) dockerd_log_path = os.path.join(repo_path, "tests/integration/dockerd.log") if os.path.exists(dockerd_log_path): - new_name = test_group_str + "_" + str(i) + "_" + os.path.basename(dockerd_log_path) - os.rename(dockerd_log_path, os.path.join(repo_path, "tests/integration", new_name)) + new_name = ( + test_group_str + + "_" + + str(i) + + "_" + + os.path.basename(dockerd_log_path) + ) + os.rename( + dockerd_log_path, + os.path.join(repo_path, "tests/integration", new_name), + ) extra_logs_names.append(new_name) if os.path.exists(info_path): extra_logs_names.append(info_basename) new_counters = get_counters(info_path) for state, tests in new_counters.items(): - logging.info("Tests with %s state (%s): %s", state, len(tests), tests) + logging.info( + "Tests with %s state (%s): %s", state, len(tests), tests + ) times_lines = parse_test_times(info_path) new_tests_times = get_test_times(times_lines) self._update_counters(counters, new_counters) @@ -512,19 +656,35 @@ class ClickhouseIntegrationTestsRunner: tests_times[test_name] = test_time test_data_dirs_new = self._find_test_data_dirs(repo_path, test_names) - test_data_dirs_diff = self._get_test_data_dirs_difference(test_data_dirs_new, test_data_dirs) + test_data_dirs_diff = self._get_test_data_dirs_difference( + test_data_dirs_new, test_data_dirs + ) test_data_dirs = test_data_dirs_new if extra_logs_names or test_data_dirs_diff: - extras_result_path = os.path.join(str(self.path()), "integration_run_" + test_group_str + "_" + str(i) + ".tar.gz") - self._compress_logs(os.path.join(repo_path, "tests/integration"), extra_logs_names + list(test_data_dirs_diff), extras_result_path) + extras_result_path = os.path.join( + str(self.path()), + "integration_run_" + test_group_str + "_" + str(i) + ".tar.gz", + ) + self._compress_logs( + os.path.join(repo_path, "tests/integration"), + extra_logs_names + list(test_data_dirs_diff), + extras_result_path, + ) log_paths.append(extras_result_path) if len(counters["PASSED"]) + len(counters["FLAKY"]) == len(tests_in_group): logging.info("All tests from group %s passed", test_group) break - if len(counters["PASSED"]) + len(counters["FLAKY"]) >= 0 and len(counters["FAILED"]) == 0 and len(counters["ERROR"]) == 0: - logging.info("Seems like all tests passed but some of them are skipped or deselected. Ignoring them and finishing group.") + if ( + len(counters["PASSED"]) + len(counters["FLAKY"]) >= 0 + and len(counters["FAILED"]) == 0 + and len(counters["ERROR"]) == 0 + ): + logging.info( + "Seems like all tests passed but some of them are skipped or " + "deselected. Ignoring them and finishing group." + ) break else: # Mark all non tried tests as errors, with '::' in name @@ -532,26 +692,28 @@ class ClickhouseIntegrationTestsRunner: # we run whole test dirs like "test_odbc_interaction" and don't # want to mark them as error so we filter by '::'. for test in tests_in_group: - if (test not in counters["PASSED"] and - test not in counters["ERROR"] and - test not in counters["SKIPPED"] and - test not in counters["FAILED"] and - '::' in test): + if ( + test not in counters["PASSED"] + and test not in counters["ERROR"] + and test not in counters["SKIPPED"] + and test not in counters["FAILED"] + and "::" in test + ): counters["ERROR"].append(test) return counters, tests_times, log_paths def run_flaky_check(self, repo_path, build_path): - pr_info = self.params['pr_info'] + pr_info = self.params["pr_info"] # pytest swears, if we require to run some tests which was renamed or deleted tests_to_run = filter_existing_tests(get_tests_to_run(pr_info), repo_path) if not tests_to_run: logging.info("No tests to run found") - return 'success', 'Nothing to run', [('Nothing to run', 'OK')], '' + return "success", "Nothing to run", [("Nothing to run", "OK")], "" self._install_clickhouse(build_path) - logging.info("Found '%s' tests to run", ' '.join(tests_to_run)) + logging.info("Found '%s' tests to run", " ".join(tests_to_run)) result_state = "success" description_prefix = "No flaky tests: " start = time.time() @@ -561,17 +723,20 @@ class ClickhouseIntegrationTestsRunner: for i in range(TRIES_COUNT): final_retry += 1 logging.info("Running tests for the %s time", i) - counters, tests_times, log_paths = self.try_run_test_group(repo_path, "flaky", tests_to_run, 1, 1) + counters, tests_times, log_paths = self.try_run_test_group( + repo_path, "flaky", tests_to_run, 1, 1 + ) logs += log_paths if counters["FAILED"]: - logging.info("Found failed tests: %s", ' '.join(counters["FAILED"])) + logging.info("Found failed tests: %s", " ".join(counters["FAILED"])) description_prefix = "Flaky tests found: " result_state = "failure" break if counters["ERROR"]: description_prefix = "Flaky tests found: " - logging.info("Found error tests: %s", ' '.join(counters["ERROR"])) - # NOTE "error" result state will restart the whole test task, so we use "failure" here + logging.info("Found error tests: %s", " ".join(counters["ERROR"])) + # NOTE "error" result state will restart the whole test task, + # so we use "failure" here result_state = "failure" break assert len(counters["FLAKY"]) == 0 @@ -591,8 +756,20 @@ class ClickhouseIntegrationTestsRunner: text_state = "FAIL" else: text_state = state - test_result += [(c + ' (✕' + str(final_retry) + ')', text_state, "{:.2f}".format(tests_times[c])) for c in counters[state]] - status_text = description_prefix + ', '.join([str(n).lower().replace('failed', 'fail') + ': ' + str(len(c)) for n, c in counters.items()]) + test_result += [ + ( + c + " (✕" + str(final_retry) + ")", + text_state, + "{:.2f}".format(tests_times[c]), + ) + for c in counters[state] + ] + status_text = description_prefix + ", ".join( + [ + str(n).lower().replace("failed", "fail") + ": " + str(len(c)) + for n, c in counters.items() + ] + ) return result_state, status_text, test_result, logs @@ -601,7 +778,10 @@ class ClickhouseIntegrationTestsRunner: return self.run_flaky_check(repo_path, build_path) self._install_clickhouse(build_path) - logging.info("Dump iptables before run %s", subprocess.check_output("sudo iptables -L", shell=True)) + logging.info( + "Dump iptables before run %s", + subprocess.check_output("sudo iptables -L", shell=True), + ) all_tests = self._get_all_tests(repo_path) if self.run_by_hash_total != 0: @@ -613,18 +793,36 @@ class ClickhouseIntegrationTestsRunner: all_tests = all_filtered_by_hash_tests parallel_skip_tests = self._get_parallel_tests_skip_list(repo_path) - logging.info("Found %s tests first 3 %s", len(all_tests), ' '.join(all_tests[:3])) - filtered_sequential_tests = list(filter(lambda test: test in all_tests, parallel_skip_tests)) - filtered_parallel_tests = list(filter(lambda test: test not in parallel_skip_tests, all_tests)) - not_found_tests = list(filter(lambda test: test not in all_tests, parallel_skip_tests)) - logging.info("Found %s tests first 3 %s, parallel %s, other %s", len(all_tests), ' '.join(all_tests[:3]), len(filtered_parallel_tests), len(filtered_sequential_tests)) - logging.info("Not found %s tests first 3 %s", len(not_found_tests), ' '.join(not_found_tests[:3])) + logging.info( + "Found %s tests first 3 %s", len(all_tests), " ".join(all_tests[:3]) + ) + filtered_sequential_tests = list( + filter(lambda test: test in all_tests, parallel_skip_tests) + ) + filtered_parallel_tests = list( + filter(lambda test: test not in parallel_skip_tests, all_tests) + ) + not_found_tests = list( + filter(lambda test: test not in all_tests, parallel_skip_tests) + ) + logging.info( + "Found %s tests first 3 %s, parallel %s, other %s", + len(all_tests), + " ".join(all_tests[:3]), + len(filtered_parallel_tests), + len(filtered_sequential_tests), + ) + logging.info( + "Not found %s tests first 3 %s", + len(not_found_tests), + " ".join(not_found_tests[:3]), + ) grouped_tests = self.group_test_by_file(filtered_sequential_tests) i = 0 for par_group in chunks(filtered_parallel_tests, PARALLEL_GROUP_SIZE): - grouped_tests["parallel{}".format(i)] = par_group - i+=1 + grouped_tests[f"parallel{i}"] = par_group + i += 1 logging.info("Found %s tests groups", len(grouped_tests)) counters = { @@ -646,12 +844,18 @@ class ClickhouseIntegrationTestsRunner: for group, tests in items_to_run: logging.info("Running test group %s containing %s tests", group, len(tests)) - group_counters, group_test_times, log_paths = self.try_run_test_group(repo_path, group, tests, MAX_RETRY, NUM_WORKERS) + group_counters, group_test_times, log_paths = self.try_run_test_group( + repo_path, group, tests, MAX_RETRY, NUM_WORKERS + ) total_tests = 0 for counter, value in group_counters.items(): - logging.info("Tests from group %s stats, %s count %s", group, counter, len(value)) + logging.info( + "Tests from group %s stats, %s count %s", group, counter, len(value) + ) counters[counter] += value - logging.info("Totally have %s with status %s", len(counters[counter]), counter) + logging.info( + "Totally have %s with status %s", len(counters[counter]), counter + ) total_tests += len(counters[counter]) logging.info("Totally finished tests %s/%s", total_tests, len(all_tests)) @@ -664,7 +868,9 @@ class ClickhouseIntegrationTestsRunner: break if counters["FAILED"] or counters["ERROR"]: - logging.info("Overall status failure, because we have tests in FAILED or ERROR state") + logging.info( + "Overall status failure, because we have tests in FAILED or ERROR state" + ) result_state = "failure" else: logging.info("Overall success!") @@ -678,42 +884,49 @@ class ClickhouseIntegrationTestsRunner: text_state = "FAIL" else: text_state = state - test_result += [(c, text_state, "{:.2f}".format(tests_times[c]), tests_log_paths[c]) for c in counters[state]] + test_result += [ + (c, text_state, "{:.2f}".format(tests_times[c]), tests_log_paths[c]) + for c in counters[state] + ] - failed_sum = len(counters['FAILED']) + len(counters['ERROR']) - status_text = "fail: {}, passed: {}, flaky: {}".format(failed_sum, len(counters['PASSED']), len(counters['FLAKY'])) + failed_sum = len(counters["FAILED"]) + len(counters["ERROR"]) + status_text = "fail: {}, passed: {}, flaky: {}".format( + failed_sum, len(counters["PASSED"]), len(counters["FLAKY"]) + ) if self.soft_deadline_time < time.time(): status_text = "Timeout, " + status_text result_state = "failure" - counters['FLAKY'] = [] + counters["FLAKY"] = [] if not counters or sum(len(counter) for counter in counters.values()) == 0: status_text = "No tests found for some reason! It's a bug" result_state = "failure" - if '(memory)' in self.params['context_name']: + if "(memory)" in self.params["context_name"]: result_state = "success" return result_state, status_text, test_result, [] + def write_results(results_file, status_file, results, status): - with open(results_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(results_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerows(results) - with open(status_file, 'w') as f: - out = csv.writer(f, delimiter='\t') + with open(status_file, "w") as f: + out = csv.writer(f, delimiter="\t") out.writerow(status) + if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s") repo_path = os.environ.get("CLICKHOUSE_TESTS_REPO_PATH") build_path = os.environ.get("CLICKHOUSE_TESTS_BUILD_PATH") result_path = os.environ.get("CLICKHOUSE_TESTS_RESULT_PATH") params_path = os.environ.get("CLICKHOUSE_TESTS_JSON_PARAMS_PATH") - params = json.loads(open(params_path, 'r').read()) + params = json.loads(open(params_path, "r").read()) runner = ClickhouseIntegrationTestsRunner(result_path, params) logging.info("Running tests") diff --git a/tests/integration/runner b/tests/integration/runner index 3687ca4068c..737eaeef683 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -238,6 +238,8 @@ if __name__ == "__main__": env_tags += "-e {}={} ".format("DOCKER_POSTGRESQL_JAVA_CLIENT_TAG", tag) elif image == "clickhouse/integration-test": env_tags += "-e {}={} ".format("DOCKER_BASE_TAG", tag) + elif image == "clickhouse/kerberized-hadoop": + env_tags += "-e {}={} ".format("DOCKER_KERBERIZED_HADOOP_TAG", tag) elif image == "clickhouse/kerberos-kdc": env_tags += "-e {}={} ".format("DOCKER_KERBEROS_KDC_TAG", tag) else: diff --git a/tests/integration/test_consistent_parts_after_clone_replica/test.py b/tests/integration/test_consistent_parts_after_clone_replica/test.py index 35a42b6fb12..b0b69da0902 100644 --- a/tests/integration/test_consistent_parts_after_clone_replica/test.py +++ b/tests/integration/test_consistent_parts_after_clone_replica/test.py @@ -3,8 +3,6 @@ import pytest from helpers.cluster import ClickHouseCluster from helpers.network import PartitionManager from helpers.test_tools import assert_eq_with_retry -import time - def fill_nodes(nodes, shard): for node in nodes: @@ -59,20 +57,25 @@ def test_inconsistent_parts_if_drop_while_replica_not_active(start_cluster): # DROP_RANGE will be removed from the replication log and the first replica will be lost for i in range(20): node2.query("INSERT INTO test_table VALUES ('2019-08-16', {})".format(20 + i)) - + assert_eq_with_retry(node2, "SELECT value FROM system.zookeeper WHERE path='/clickhouse/tables/test1/replicated/replicas/node1' AND name='is_lost'", "1") - for i in range(30): - if node2.contains_in_log("Will mark replica node1 as lost"): - break - time.sleep(0.5) + node2.wait_for_log_line("Will mark replica node1 as lost") # the first replica will be cloned from the second pm.heal_all() + node2.wait_for_log_line("Sending part") assert_eq_with_retry(node1, "SELECT count(*) FROM test_table", node2.query("SELECT count(*) FROM test_table")) # ensure replica was cloned assert node1.contains_in_log("Will mimic node2") - # queue must be empty (except some merges that are possibly executing right now) - assert node1.query("SELECT count() FROM system.replication_queue WHERE type != 'MERGE_PARTS'") == "0\n" - assert node2.query("SELECT count() FROM system.replication_queue WHERE type != 'MERGE_PARTS'") == "0\n" + + # 2 options: + # - There wasn't a merge in node2. Then node1 should have cloned the 2 parts + # - There was a merge in progress. node1 might have cloned the new part but still has the original 2 parts + # in the replication queue until they are finally discarded with a message like: + # `Skipping action for part 201908_40_40_0 because part 201908_21_40_4 already exists.` + # + # In any case after a short while the replication queue should be empty + assert_eq_with_retry(node1, "SELECT count() FROM system.replication_queue WHERE type != 'MERGE_PARTS'", "0") + assert_eq_with_retry(node2, "SELECT count() FROM system.replication_queue WHERE type != 'MERGE_PARTS'", "0") diff --git a/tests/integration/test_distributed_insert_backward_compatibility/__init__.py b/tests/integration/test_distributed_insert_backward_compatibility/__init__.py new file mode 100644 index 00000000000..8b137891791 --- /dev/null +++ b/tests/integration/test_distributed_insert_backward_compatibility/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/integration/test_distributed_insert_backward_compatibility/configs/remote_servers.xml b/tests/integration/test_distributed_insert_backward_compatibility/configs/remote_servers.xml new file mode 100644 index 00000000000..9c7f02c190f --- /dev/null +++ b/tests/integration/test_distributed_insert_backward_compatibility/configs/remote_servers.xml @@ -0,0 +1,12 @@ + + + + + + node1 + 9000 + + + + + diff --git a/tests/integration/test_distributed_insert_backward_compatibility/test.py b/tests/integration/test_distributed_insert_backward_compatibility/test.py new file mode 100644 index 00000000000..ba7d8e0a25d --- /dev/null +++ b/tests/integration/test_distributed_insert_backward_compatibility/test.py @@ -0,0 +1,39 @@ +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.client import QueryRuntimeException + +cluster = ClickHouseCluster(__file__) + +node_shard = cluster.add_instance('node1', main_configs=['configs/remote_servers.xml']) + +node_dist = cluster.add_instance('node2', main_configs=['configs/remote_servers.xml'], image='yandex/clickhouse-server', + tag='21.11.9.1', stay_alive=True, with_installed_binary=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + node_shard.query("CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id") + node_dist.query("CREATE TABLE local_table(id UInt32, val String) ENGINE = MergeTree ORDER BY id") + node_dist.query("CREATE TABLE dist_table(id UInt32, val String) ENGINE = Distributed(test_cluster, default, local_table, rand())") + + yield cluster + + finally: + cluster.shutdown() + + +def test_distributed_in_tuple(started_cluster): + node_dist.query("SYSTEM STOP DISTRIBUTED SENDS dist_table") + + node_dist.query("INSERT INTO dist_table VALUES (1, 'foo')") + assert node_dist.query("SELECT count() FROM dist_table") == "0\n" + assert node_shard.query("SELECT count() FROM local_table") == "0\n" + + node_dist.restart_with_latest_version(signal=9) + node_dist.query("SYSTEM FLUSH DISTRIBUTED dist_table") + + assert node_dist.query("SELECT count() FROM dist_table") == "1\n" + assert node_shard.query("SELECT count() FROM local_table") == "1\n" diff --git a/tests/integration/test_distributed_inter_server_secret/test.py b/tests/integration/test_distributed_inter_server_secret/test.py index 73d338ba870..2601163d790 100644 --- a/tests/integration/test_distributed_inter_server_secret/test.py +++ b/tests/integration/test_distributed_inter_server_secret/test.py @@ -87,6 +87,17 @@ def get_query_user_info(node, query_pattern): type = 'QueryFinish' """.format(query_pattern)).strip().split('\t') +# @return -- [user, initial_user] +def get_query_user_info_by_id(node, query_id): + node.query("SYSTEM FLUSH LOGS") + return node.query(""" + SELECT user, initial_user + FROM system.query_log + WHERE + query_id = '{}' AND + type = 'QueryFinish' + """.format(query_id)).strip().split('\t') + # @return -- settings def get_query_setting_on_shard(node, query_pattern, setting): node.query("SYSTEM FLUSH LOGS") @@ -183,6 +194,7 @@ def test_secure_insert_buffer_async(): # previous connection that was instantiated with "ro" user (using # interserver secret) assert not n1.contains_in_log('{' + query_id + '} Connection (n2:9000): Connecting.') + assert get_query_user_info_by_id(n1, query_id) == ['default', 'default'] # And before the bug was fixed this query will fail with the following error: # @@ -191,6 +203,18 @@ def test_secure_insert_buffer_async(): n1.query('OPTIMIZE TABLE dist_secure_buffer') n1.query('SYSTEM FLUSH DISTRIBUTED ON CLUSTER secure dist_secure_from_buffer') + # Check user from which the INSERT on the remote node will be executed + # + # Incorrect example: + # + # {2c55669f-71ad-48fe-98fa-7b475b80718e} executeQuery: (from 172.16.1.1:44636, user: ro) INSERT INTO default.data_from_buffer (key) VALUES + # + # Correct example: + # + # {2c55669f-71ad-48fe-98fa-7b475b80718e} executeQuery: (from 0.0.0.0:0, user: ) INSERT INTO default.data_from_buffer (key) VALUES + # + assert n2.contains_in_log('executeQuery: (from 0.0.0.0:0, user: ) INSERT INTO default.data_from_buffer (key) VALUES') + assert int(n1.query('SELECT count() FROM dist_secure_from_buffer')) == 2 n1.query('TRUNCATE TABLE data_from_buffer ON CLUSTER secure') diff --git a/tests/integration/test_grant_and_revoke/test.py b/tests/integration/test_grant_and_revoke/test.py index b905e4df219..196141f9bfe 100644 --- a/tests/integration/test_grant_and_revoke/test.py +++ b/tests/integration/test_grant_and_revoke/test.py @@ -150,8 +150,11 @@ def test_grant_all_on_table(): instance.query("CREATE USER A, B") instance.query("GRANT ALL ON test.table TO A WITH GRANT OPTION") instance.query("GRANT ALL ON test.table TO B", user='A') - assert instance.query( - "SHOW GRANTS FOR B") == "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n" + assert instance.query("SHOW GRANTS FOR B") ==\ + "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, "\ + "DROP TABLE, DROP VIEW, DROP DICTIONARY, TRUNCATE, OPTIMIZE, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, "\ + "SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, "\ + "SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM FLUSH DISTRIBUTED, dictGet ON test.table TO B\n" instance.query("REVOKE ALL ON test.table FROM B", user='A') assert instance.query("SHOW GRANTS FOR B") == "" @@ -250,6 +253,15 @@ def test_introspection(): assert instance.query("SHOW GRANTS", user='A') == TSV(["GRANT SELECT ON test.table TO A"]) assert instance.query("SHOW GRANTS", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS FOR ALL", user='A') == TSV(["GRANT SELECT ON test.table TO A"]) + assert instance.query("SHOW GRANTS FOR ALL", user='B') == TSV(["GRANT CREATE ON *.* TO B WITH GRANT OPTION"]) + assert instance.query("SHOW GRANTS FOR ALL") == TSV(["GRANT SELECT ON test.table TO A", + "GRANT CREATE ON *.* TO B WITH GRANT OPTION", + "GRANT ALL ON *.* TO default WITH GRANT OPTION"]) + + expected_error = "necessary to have grant SHOW USERS" + assert expected_error in instance.query_and_get_error("SHOW GRANTS FOR B", user='A') + expected_access1 = "CREATE USER A\n" \ "CREATE USER B\n" \ "CREATE USER default IDENTIFIED WITH plaintext_password SETTINGS PROFILE default" diff --git a/tests/integration/test_grpc_protocol/test.py b/tests/integration/test_grpc_protocol/test.py index e17ed0d9c8e..bd9a0cbe438 100644 --- a/tests/integration/test_grpc_protocol/test.py +++ b/tests/integration/test_grpc_protocol/test.py @@ -2,6 +2,8 @@ import os import pytest import sys import time +import pytz +import uuid import grpc from helpers.cluster import ClickHouseCluster, run_and_check from threading import Thread @@ -43,8 +45,8 @@ def create_channel(): main_channel = channel return channel -def query_common(query_text, settings={}, input_data=[], input_data_delimiter='', output_format='TabSeparated', external_tables=[], - user_name='', password='', query_id='123', session_id='', stream_output=False, channel=None): +def query_common(query_text, settings={}, input_data=[], input_data_delimiter='', output_format='TabSeparated', send_output_columns=False, + external_tables=[], user_name='', password='', query_id='123', session_id='', stream_output=False, channel=None): if type(input_data) is not list: input_data = [input_data] if type(input_data_delimiter) is str: @@ -58,7 +60,8 @@ def query_common(query_text, settings={}, input_data=[], input_data_delimiter='' input_data_part = input_data_part.encode(DEFAULT_ENCODING) return clickhouse_grpc_pb2.QueryInfo(query=query_text, settings=settings, input_data=input_data_part, input_data_delimiter=input_data_delimiter, output_format=output_format, - external_tables=external_tables, user_name=user_name, password=password, query_id=query_id, + send_output_columns=send_output_columns, external_tables=external_tables, + user_name=user_name, password=password, query_id=query_id, session_id=session_id, next_query_info=bool(input_data)) def send_query_info(): yield query_info() @@ -177,13 +180,13 @@ def test_insert_query_delimiter(): assert query("SELECT a FROM t ORDER BY a") == "1\n5\n234\n" def test_insert_default_column(): - query("CREATE TABLE t (a UInt8, b Int32 DEFAULT 100, c String DEFAULT 'c') ENGINE = Memory") + query("CREATE TABLE t (a UInt8, b Int32 DEFAULT 100 - a, c String DEFAULT 'c') ENGINE = Memory") query("INSERT INTO t (c, a) VALUES ('x',1),('y',2)") query("INSERT INTO t (a) FORMAT TabSeparated", input_data="3\n4\n") - assert query("SELECT * FROM t ORDER BY a") == "1\t100\tx\n" \ - "2\t100\ty\n" \ - "3\t100\tc\n" \ - "4\t100\tc\n" + assert query("SELECT * FROM t ORDER BY a") == "1\t99\tx\n" \ + "2\t98\ty\n" \ + "3\t97\tc\n" \ + "4\t96\tc\n" def test_insert_splitted_row(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") @@ -204,6 +207,28 @@ def test_totals_and_extremes(): assert query("SELECT x, y FROM t") == "1\t2\n2\t4\n3\t2\n3\t3\n3\t4\n" assert query_and_get_extremes("SELECT x, y FROM t", settings={"extremes": "1"}) == "1\t2\n3\t4\n" +def test_get_query_details(): + result = list(query_no_errors("CREATE TABLE t (a UInt8) ENGINE = Memory", query_id = '123'))[0] + assert result.query_id == '123' + pytz.timezone(result.time_zone) + assert result.output_format == '' + assert len(result.output_columns) == 0 + assert result.output == b'' + # + result = list(query_no_errors("SELECT 'a', 1", query_id = '', output_format = 'TabSeparated'))[0] + uuid.UUID(result.query_id) + pytz.timezone(result.time_zone) + assert result.output_format == 'TabSeparated' + assert len(result.output_columns) == 0 + assert result.output == b'a\t1\n' + # + result = list(query_no_errors("SELECT 'a' AS x, 1 FORMAT JSONEachRow", query_id = '', send_output_columns=True))[0] + uuid.UUID(result.query_id) + pytz.timezone(result.time_zone) + assert result.output_format == 'JSONEachRow' + assert ([(col.name, col.type) for col in result.output_columns]) == [('x', 'String'), ('1', 'UInt8')] + assert result.output == b'{"x":"a","1":1}\n' + def test_errors_handling(): e = query_and_get_error("") #print(e) @@ -225,6 +250,9 @@ def test_logs(): def test_progress(): results = query_no_errors("SELECT number, sleep(0.31) FROM numbers(8) SETTINGS max_block_size=2, interactive_delay=100000", stream_output=True) + for result in results: + result.time_zone = '' + result.query_id = '' #print(results) assert str(results) ==\ """[progress { @@ -232,6 +260,7 @@ def test_progress(): read_bytes: 16 total_rows_to_read: 8 } +output_format: "TabSeparated" , output: "0\\t0\\n1\\t0\\n" , progress { read_rows: 2 @@ -257,7 +286,7 @@ def test_progress(): } ]""" -def test_session(): +def test_session_settings(): session_a = "session A" session_b = "session B" query("SET custom_x=1", session_id=session_a) @@ -267,9 +296,22 @@ def test_session(): assert query("SELECT getSetting('custom_x'), getSetting('custom_y')", session_id=session_a) == "1\t2\n" assert query("SELECT getSetting('custom_x'), getSetting('custom_y')", session_id=session_b) == "3\t4\n" +def test_session_temp_tables(): + session_a = "session A" + session_b = "session B" + query("CREATE TEMPORARY TABLE my_temp_table(a Int8)", session_id=session_a) + query("INSERT INTO my_temp_table VALUES (10)", session_id=session_a) + assert query("SELECT * FROM my_temp_table", session_id=session_a) == "10\n" + query("CREATE TEMPORARY TABLE my_temp_table(a Int8)", session_id=session_b) + query("INSERT INTO my_temp_table VALUES (20)", session_id=session_b) + assert query("SELECT * FROM my_temp_table", session_id=session_b) == "20\n" + assert query("SELECT * FROM my_temp_table", session_id=session_a) == "10\n" + def test_no_session(): e = query_and_get_error("SET custom_x=1") assert "There is no session" in e.display_text + e = query_and_get_error("CREATE TEMPORARY TABLE my_temp_table(a Int8)") + assert "There is no session" in e.display_text def test_input_function(): query("CREATE TABLE t (a UInt8) ENGINE = Memory") @@ -360,22 +402,14 @@ def test_cancel_while_generating_output(): output += result.output assert output == b'0\t0\n1\t0\n2\t0\n3\t0\n' -def test_result_compression(): - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000000)", - result_compression=clickhouse_grpc_pb2.Compression(algorithm=clickhouse_grpc_pb2.CompressionAlgorithm.GZIP, - level=clickhouse_grpc_pb2.CompressionLevel.COMPRESSION_HIGH)) - stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) - result = stub.ExecuteQuery(query_info) - assert result.output == (b'0\n')*1000000 - def test_compressed_output(): - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000)", compression_type="lz4") + query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000)", output_compression_type="lz4") stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQuery(query_info) assert lz4.frame.decompress(result.output) == (b'0\n')*1000 def test_compressed_output_streaming(): - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(100000)", compression_type="lz4") + query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(100000)", output_compression_type="lz4") stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) d_context = lz4.frame.create_decompression_context() data = b'' @@ -385,7 +419,7 @@ def test_compressed_output_streaming(): assert data == (b'0\n')*100000 def test_compressed_output_gzip(): - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000)", compression_type="gzip", compression_level=6) + query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000)", output_compression_type="gzip", output_compression_level=6) stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) result = stub.ExecuteQuery(query_info) assert gzip.decompress(result.output) == (b'0\n')*1000 @@ -394,10 +428,10 @@ def test_compressed_totals_and_extremes(): query("CREATE TABLE t (x UInt8, y UInt8) ENGINE = Memory") query("INSERT INTO t VALUES (1, 2), (2, 4), (3, 2), (3, 3), (3, 4)") stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT sum(x), y FROM t GROUP BY y WITH TOTALS", compression_type="lz4") + query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT sum(x), y FROM t GROUP BY y WITH TOTALS", output_compression_type="lz4") result = stub.ExecuteQuery(query_info) assert lz4.frame.decompress(result.totals) == b'12\t0\n' - query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT x, y FROM t", settings={"extremes": "1"}, compression_type="lz4") + query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT x, y FROM t", settings={"extremes": "1"}, output_compression_type="lz4") result = stub.ExecuteQuery(query_info) assert lz4.frame.decompress(result.extremes) == b'1\t2\n3\t4\n' @@ -410,7 +444,7 @@ def test_compressed_insert_query_streaming(): d2 = data[sz1:sz1+sz2] d3 = data[sz1+sz2:] def send_query_info(): - yield clickhouse_grpc_pb2.QueryInfo(query="INSERT INTO t VALUES", input_data=d1, compression_type="lz4", next_query_info=True) + yield clickhouse_grpc_pb2.QueryInfo(query="INSERT INTO t VALUES", input_data=d1, input_compression_type="lz4", next_query_info=True) yield clickhouse_grpc_pb2.QueryInfo(input_data=d2, next_query_info=True) yield clickhouse_grpc_pb2.QueryInfo(input_data=d3) stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) @@ -431,3 +465,24 @@ def test_compressed_external_table(): b"3\tCarl\n"\ b"4\tDaniel\n"\ b"5\tEthan\n" + +def test_transport_compression(): + query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 0 FROM numbers(1000000)", transport_compression_type='gzip', transport_compression_level=3) + stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) + result = stub.ExecuteQuery(query_info) + assert result.output == (b'0\n')*1000000 + +def test_opentelemetry_context_propagation(): + trace_id = "80c190b5-9dc1-4eae-82b9-6c261438c817" + parent_span_id = 123 + trace_state = "some custom state" + trace_id_hex = trace_id.replace("-", "") + parent_span_id_hex = f'{parent_span_id:0>16X}' + metadata = [("traceparent", f"00-{trace_id_hex}-{parent_span_id_hex}-01"), ("tracestate", trace_state)] + stub = clickhouse_grpc_pb2_grpc.ClickHouseStub(main_channel) + query_info = clickhouse_grpc_pb2.QueryInfo(query="SELECT 1") + result = stub.ExecuteQuery(query_info, metadata=metadata) + assert result.output == b"1\n" + node.query("SYSTEM FLUSH LOGS") + assert node.query(f"SELECT attribute['db.statement'], attribute['clickhouse.tracestate'] FROM system.opentelemetry_span_log " + f"WHERE trace_id='{trace_id}' AND parent_span_id={parent_span_id}") == "SELECT 1\tsome custom state\n" diff --git a/src/Functions/DateOrDateTimeFunctionsConvertion.cpp b/tests/integration/test_jbod_ha/__init__.py similarity index 100% rename from src/Functions/DateOrDateTimeFunctionsConvertion.cpp rename to tests/integration/test_jbod_ha/__init__.py diff --git a/tests/integration/test_jbod_ha/configs/config.d/storage_configuration.xml b/tests/integration/test_jbod_ha/configs/config.d/storage_configuration.xml new file mode 100644 index 00000000000..b5c351d105b --- /dev/null +++ b/tests/integration/test_jbod_ha/configs/config.d/storage_configuration.xml @@ -0,0 +1,30 @@ + + 1000 + + + + 1024 + + + /jbod1/ + + + /jbod2/ + + + /jbod3/ + + + + + + + jbod1 + jbod2 + jbod3 + + + + + + diff --git a/tests/integration/test_jbod_ha/test.py b/tests/integration/test_jbod_ha/test.py new file mode 100644 index 00000000000..0a8631ff207 --- /dev/null +++ b/tests/integration/test_jbod_ha/test.py @@ -0,0 +1,111 @@ +import json +import random +import re +import string +import threading +import time +from multiprocessing.dummy import Pool + +import pytest +from helpers.client import QueryRuntimeException +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance( + "node1", + main_configs=["configs/config.d/storage_configuration.xml",], + with_zookeeper=True, + stay_alive=True, + tmpfs=["/jbod1:size=100M", "/jbod2:size=100M", "/jbod3:size=100M"], + macros={"shard": 0, "replica": 1}, +) + + +node2 = cluster.add_instance( + "node2", + main_configs=["configs/config.d/storage_configuration.xml"], + with_zookeeper=True, + stay_alive=True, + tmpfs=["/jbod1:size=100M", "/jbod2:size=100M", "/jbod3:size=100M"], + macros={"shard": 0, "replica": 2}, +) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + + finally: + cluster.shutdown() + + +def test_jbod_ha(start_cluster): + try: + for i, node in enumerate([node1, node2]): + node.query( + """ + CREATE TABLE tbl (p UInt8, d String) + ENGINE = ReplicatedMergeTree('/clickhouse/tbl', '{}') + PARTITION BY p + ORDER BY tuple() + SETTINGS + storage_policy = 'jbod', + old_parts_lifetime = 1, + cleanup_delay_period = 1, + cleanup_delay_period_random_add = 2, + max_bytes_to_merge_at_max_space_in_pool = 4096 + """.format( + i + ) + ) + + for i in range(50): + # around 1k per block + node1.query( + "insert into tbl select randConstant() % 2, randomPrintableASCII(16) from numbers(50)" + ) + + node2.query("SYSTEM SYNC REPLICA tbl", timeout=10) + + # mimic disk failure + node1.exec_in_container( + ["bash", "-c", "chmod -R 000 /jbod1"], privileged=True, user="root" + ) + + time.sleep(3) + + # after 3 seconds jbod1 will be set as broken disk. Let's wait for another 5 seconds for data to be recovered + time.sleep(5) + + assert ( + int( + node1.query("select total_space from system.disks where name = 'jbod1'") + ) + == 0 + ) + + assert int(node1.query("select count(p) from tbl")) == 2500 + + # mimic disk recovery + node1.exec_in_container( + ["bash", "-c", "chmod -R 755 /jbod1"], + privileged=True, + user="root", + ) + node1.query("system restart disk jbod1") + + time.sleep(5) + + assert ( + int( + node1.query("select total_space from system.disks where name = 'jbod1'") + ) + > 0 + ) + + finally: + for node in [node1, node2]: + node.query("DROP TABLE IF EXISTS tbl SYNC") diff --git a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py index 3f219b6ba57..9f5c7b1c8ce 100644 --- a/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py +++ b/tests/integration/test_merge_tree_s3_failover/s3_endpoint/endpoint.py @@ -1,4 +1,5 @@ from bottle import request, route, run, response +from threading import Lock # Endpoint can be configured to throw 500 error on N-th request attempt. @@ -6,6 +7,7 @@ from bottle import request, route, run, response # Dict to the number of request should be failed. cache = {} +mutex = Lock() @route('/fail_request/<_request_number>') @@ -38,23 +40,34 @@ def delete(_bucket): @route('/<_bucket>/<_path:path>', ['GET', 'POST', 'PUT', 'DELETE']) def server(_bucket, _path): - if cache.get('request_number', None): - request_number = cache.pop('request_number') - 1 - if request_number > 0: - cache['request_number'] = request_number - else: - response.status = 500 - response.content_type = 'text/xml' - return 'ExpectedErrorExpected Errortxfbd566d03042474888193-00608d7537' - if cache.get('throttle_request_number', None): - request_number = cache.pop('throttle_request_number') - 1 - if request_number > 0: - cache['throttle_request_number'] = request_number - else: - response.status = 429 - response.content_type = 'text/xml' - return 'TooManyRequestsExceptionPlease reduce your request rate.txfbd566d03042474888193-00608d7538' + # It's delete query for failed part + if _path.endswith('delete'): + response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) + response.status = 307 + return 'Redirected' + + mutex.acquire() + try: + if cache.get('request_number', None): + request_number = cache.pop('request_number') - 1 + if request_number > 0: + cache['request_number'] = request_number + else: + response.status = 500 + response.content_type = 'text/xml' + return 'ExpectedErrorExpected Errortxfbd566d03042474888193-00608d7537' + + if cache.get('throttle_request_number', None): + request_number = cache.pop('throttle_request_number') - 1 + if request_number > 0: + cache['throttle_request_number'] = request_number + else: + response.status = 429 + response.content_type = 'text/xml' + return 'TooManyRequestsExceptionPlease reduce your request rate.txfbd566d03042474888193-00608d7538' + finally: + mutex.release() response.set_header("Location", "http://minio1:9001/" + _bucket + '/' + _path) response.status = 307 diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index db541edde9c..e2b30b8f90e 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -382,12 +382,18 @@ def test_round_robin(start_cluster, name, engine): used_disk = get_used_disks_for_table(node1, name) assert len(used_disk) == 1, 'More than one disk used for single insert' + # sleep is required because we order disks by their modification time, and if insert will be fast + # modification time of two disks will be equal, then sort will not provide deterministic results + time.sleep(5) + node1.query_with_retry("insert into {} select * from numbers(10000, 10000)".format(name)) used_disks = get_used_disks_for_table(node1, name) assert len(used_disks) == 2, 'Two disks should be used for two parts' assert used_disks[0] != used_disks[1], "Should write to different disks" + time.sleep(5) + node1.query_with_retry("insert into {} select * from numbers(20000, 10000)".format(name)) used_disks = get_used_disks_for_table(node1, name) @@ -1102,8 +1108,7 @@ def test_simple_replication_and_moves(start_cluster): for task in tasks: task.get(timeout=60) - node1.query_with_retry("SYSTEM SYNC REPLICA replicated_table_for_moves", timeout=5) - node2.query_with_retry("SYSTEM SYNC REPLICA replicated_table_for_moves", timeout=5) + node1.query_with_retry("SYSTEM SYNC REPLICA ON CLUSTER test_cluster replicated_table_for_moves", timeout=5) node1.query("SELECT COUNT() FROM replicated_table_for_moves") == "40\n" node2.query("SELECT COUNT() FROM replicated_table_for_moves") == "40\n" @@ -1126,8 +1131,7 @@ def test_simple_replication_and_moves(start_cluster): disks1 = get_used_disks_for_table(node1, "replicated_table_for_moves") disks2 = get_used_disks_for_table(node2, "replicated_table_for_moves") - node1.query("SYSTEM START MERGES") - node2.query("SYSTEM START MERGES") + node2.query("SYSTEM START MERGES ON CLUSTER test_cluster") set(disks1) == set(["jbod1", "external"]) set(disks2) == set(["jbod1", "external"]) @@ -1513,6 +1517,24 @@ def test_no_merges_in_configuration_allow_from_query_with_reload(start_cluster): finally: node1.query("SYSTEM STOP MERGES ON VOLUME {}.external".format(policy)) +def test_no_merges_in_configuration_allow_from_query_with_reload_on_cluster(start_cluster): + try: + name = "test_no_merges_in_configuration_allow_from_query_with_reload" + policy = "small_jbod_with_external_no_merges" + node1.restart_clickhouse(kill=True) + assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 1] + _check_merges_are_working(node1, policy, "external", False) + + _insert_merge_execute(node1, name, policy, 2, [ + "SYSTEM START MERGES ON CLUSTER test_cluster ON VOLUME {}.external".format(policy), + "SYSTEM RELOAD CONFIG ON CLUSTER test_cluster" + ], 2, 1) + assert _get_prefer_not_to_merge_for_storage_policy(node1, policy) == [0, 0] + _check_merges_are_working(node1, policy, "external", True) + + finally: + node1.query("SYSTEM STOP MERGES ON CLUSTER test_cluster ON VOLUME {}.external".format(policy)) + def test_yes_merges_in_configuration_disallow_from_query_without_reload(start_cluster): try: diff --git a/tests/integration/test_quota/test.py b/tests/integration/test_quota/test.py index 4149987996b..83ee32bd7dd 100644 --- a/tests/integration/test_quota/test.py +++ b/tests/integration/test_quota/test.py @@ -94,9 +94,9 @@ def test_quota_from_users_xml(): system_quota_usage( [["myQuota", "default", 31556952, 1, 1000, 1, 500, 0, 500, 0, "\\N", 50, "\\N", 200, "\\N", 50, 1000, 200, "\\N", "\\N"]]) - instance.query("SELECT COUNT() from test_table") + instance.query("SELECT SUM(x) from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 2, 1000, 2, 500, 0, 500, 0, "\\N", 51, "\\N", 208, "\\N", 50, 1000, 200, "\\N", "\\N"]]) + [["myQuota", "default", 31556952, 2, 1000, 2, 500, 0, 500, 0, "\\N", 51, "\\N", 208, "\\N", 100, 1000, 400, "\\N", "\\N"]]) def test_simpliest_quota(): @@ -125,9 +125,9 @@ def test_tracking_quota(): system_quota_usage( [["myQuota", "default", 31556952, 1, "\\N", 1, "\\N", 0, "\\N", 0, "\\N", 50, "\\N", 200, "\\N", 50, "\\N", 200, "\\N", "\\N"]]) - instance.query("SELECT COUNT() from test_table") + instance.query("SELECT SUM(x) from test_table") system_quota_usage( - [["myQuota", "default", 31556952, 2, "\\N", 2, "\\N", 0, "\\N", 0, "\\N", 51, "\\N", 208, "\\N", 50, "\\N", 200, "\\N", "\\N"]]) + [["myQuota", "default", 31556952, 2, "\\N", 2, "\\N", 0, "\\N", 0, "\\N", 51, "\\N", 208, "\\N", 100, "\\N", 400, "\\N", "\\N"]]) def test_exceed_quota(): diff --git a/tests/integration/test_reload_certificate/__init__.py b/tests/integration/test_reload_certificate/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/integration/test_reload_certificate/configs/cert.xml b/tests/integration/test_reload_certificate/configs/cert.xml new file mode 100644 index 00000000000..91fe8156d1e --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/cert.xml @@ -0,0 +1,14 @@ + + + 8443 + + + /etc/clickhouse-server/config.d/first.crt + /etc/clickhouse-server/config.d/first.key + true + true + sslv2,sslv3 + true + + + diff --git a/tests/integration/test_reload_certificate/configs/first.crt b/tests/integration/test_reload_certificate/configs/first.crt new file mode 100644 index 00000000000..b0c4c3e5adf --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/first.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfOgAwIBAgIUcA+y3LQyfpxlBzL7IQVKUfnhRncwDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTIyMDEyODExNTA1NloYDzIzMTIw +NDE4MTE1MDU2WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQCumguFo+M0AQ6SgyL8K2Kep3YFYaJUjU3Mm7rrYXP7 +aGnm0Cvh3dqituSHF1ZoLgThqdBR4e/e5SZvS7ShCnFCBZpWSuhodp7qZy4ETqa8 +1/TJUr2hQLH+GpldAeGxPuDJwsdEEk100l4UHWQYg0+kqAmkijWlXDrxJYzeZ5Q5 +r/qxJN1kGxFnGYtlFjM3IpunXsREjcxjJmE4pDHp+Bkvkp0znajPJo8AE4pZ0zEQ +K/LfQSHh5BWSLw3SwGzHTTsHkn7KduaIppbYCG1j8/VEGKJIZMWUP4UbBfZ5Pl1+ +tm7sPOKho+pu35pA/7keYEP1XxGSbHy4e8xO2DkIDSBRAgMBAAGjUzBRMB0GA1Ud +DgQWBBQQ1RwP+LO8L9WmeW9xijBbG93jTjAfBgNVHSMEGDAWgBQQ1RwP+LO8L9Wm +eW9xijBbG93jTjAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAV +hqUMF75V8dEqCF61GsHizcL0R+KN36AjFckzZfMCk/H5UFIQ2L6vCsm8tmi6DlZQ +4VXlq+gCuAXuKWDe5yGNBxveKgdIltCJ85ZcySU1KON0OJIICW0lmOsFflqjD6cZ +tnP0FvFgoRRHW+lEQcgklXDF4taV9cF40xhNQ+TnzXSEmeg9fJQeC2rKLdwbpSAe +xfFPUvaKTF6w4caUqBojvTk0eKahva+kEtoeJ6KKPoBxkJsUzdL63V7P6FlktWut +0O3H4oFTZ6fNYKBKRwYeSZY30o8N6Lib0LihScLF7HWQNs0eyQ6m6v93KcJyK42F +wweXxFoWbdsITFYOdBmD +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_certificate/configs/first.key b/tests/integration/test_reload_certificate/configs/first.key new file mode 100644 index 00000000000..a66bef9ad98 --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/first.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCumguFo+M0AQ6S +gyL8K2Kep3YFYaJUjU3Mm7rrYXP7aGnm0Cvh3dqituSHF1ZoLgThqdBR4e/e5SZv +S7ShCnFCBZpWSuhodp7qZy4ETqa81/TJUr2hQLH+GpldAeGxPuDJwsdEEk100l4U +HWQYg0+kqAmkijWlXDrxJYzeZ5Q5r/qxJN1kGxFnGYtlFjM3IpunXsREjcxjJmE4 +pDHp+Bkvkp0znajPJo8AE4pZ0zEQK/LfQSHh5BWSLw3SwGzHTTsHkn7KduaIppbY +CG1j8/VEGKJIZMWUP4UbBfZ5Pl1+tm7sPOKho+pu35pA/7keYEP1XxGSbHy4e8xO +2DkIDSBRAgMBAAECggEAZSiW2Fy1fCHIoZYcpOE2CBmZxVBlznr3wj3PtCQIIHbE +NJgTdI8m5vLzwFkDFOTkqyHJskcmxIsbE4xXIJ5+M/QvESPhNvTS6ZfSD2jKLcso +5aNsfoqPFVuv0zUN37VAY2TYMlYwTii7nQfSQGmDsTAyNgRlRGMFO0W4Mfrs4+Zd +ysoxdb+562DfKnqzTaqWIGXB7kW4bdUmQwK5dCmuj4m5yh0TknPM2w+jtI/O5mA9 +pTG8p/te8b8qkrVaPyrApVNuBEonIOBfesFjnvjqIMquCutysii/hMkP6LbkXE+0 +bpcVV8Rs1W0I1zU6veAh3ValDYpiWRGX/9IALfIMAQKBgQDjEqK0Qzh8kMhO2g2E ++zh32ZsMmt7/wmhYhfmunhYKgjFyLVmAsDVHDoC/wtimItljSdKJdPenBKloTzxh +L/EuP5Fqt6BIlkrwdiXurTXEVWrntNenzpUBxGeXSmJ4B4BFJhNpQj1ewASuKjrM +CrIwwhIJRq0MjsG8aOwWHOYNMQKBgQDE2DU+jjHN12rvF+68+8HE6Gx/op6POSiW +Jb+IJRFGSrntLEXRQgvj6Tybm+dbrTy+OtXAZRo7W9hUjf5eYav1lnLj/YguvAhy +/9x97edZ9CJjvW/fEpkRBXdNkvKfqaR8qQaQSlAJRu6syreXJbPgAQOwQWVIXGa8 +N+TGIhz9IQKBgQCfLvY+to0HzhuOI5Css8yPQE5QlNVVqHyr6ifyAMLk1QZCy4Xe +ECkZShJ52+cy+GU7FIpycDwYqszz4fArFYfW6xtPG7FSkYGxdrH60xRJMbRDAOTZ +r5mH5p7UUYIcMO38C8g51wTcwnHFgrc7SRhH1BT+ybwQfJdWNJukmNexUQKBgQCI +eRHpLfKvsMNtwtz9X1qHZ1EZ6KgfylQuTTuOa4yffF2NZt186FqQB/vCMwPjVqc/ +iFD8E9xs/Q9uCAgsbXEoUseS9Ar/w9PjzyqSkGeOwSk6l3NBaIaA+5YsTU4zjg0B +dLqdPThiRjBh0iYY/8XG700cXSqYUZ/UrLfK+om4oQKBgBWSH0guEZyx37kzrN9z +bIsjSgkYxuoD18fpifAyVDmJGhi3JK4fSNz9gvsCEpPTVYMh2++HB2Pm45oNeO/m +XxqJQS+/tOcFxLC3Goy274jH6LK+3AXE++W2jV/G9Rkgv3et7KiKXbwWnYW3qIA0 +Sgm0PSmXIcMhsb03LCqfsQa1 +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_certificate/configs/second.crt b/tests/integration/test_reload_certificate/configs/second.crt new file mode 100644 index 00000000000..165ddc1959d --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/second.crt @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDCzCCAfOgAwIBAgIUc5pIrYIv905gvuYtD/Y4LPDeXKswDQYJKoZIhvcNAQEL +BQAwFDESMBAGA1UEAwwJbG9jYWxob3N0MCAXDTIyMDEyODExNTE0NVoYDzIzMTIw +NDE4MTE1MTQ1WjAUMRIwEAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEB +AQUAA4IBDwAwggEKAoIBAQDrQQCvvoSpNjl1HvPgI9ST+UbjZGEYHLVPlLTVh/Q/ +RIQg/zxVOWmuHOnFXRAyb2b7B8uuNBC8wzHaubLkP6wk8nBbhc5kK4ohhiFA3DD8 +6DGXpxfZhlZ/x/mnQnb8T+PFSPXNfJxTer1RttBBzHSiRcG0cTkCPH0oIYBRbNAO +Ig7/76EGHhNNBLDgU7CaMpvOeefMVJ1qd5SYRDgLRvZa4Y3KWtA9WrTQmDTH6YzH ++bLnVBfUV5rs9nyM4B8pGNrezb/deFlzB9c8+FhFxxm8UjeOZZhzyro+7eToVpDf +btuYcmjgju7O3142/s2P29RTogCteA8PP4KHc6oekztDAgMBAAGjUzBRMB0GA1Ud +DgQWBBQ5/fotxXkvhLkBUMfzjurEbwukOzAfBgNVHSMEGDAWgBQ5/fotxXkvhLkB +UMfzjurEbwukOzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQB+ +PWMgxHoN0xqDdLq/3cqGoO+4iG4CZGUIG7VspnojjGrHKTA/0jvC2lu62q/Dxwsv +EXL/0Gt3ycfP0pbtVIcPjB2j3u+c9iJJ2R47Q/6FRwSiN7LpUohJvCMnTDpK19wj +p5TNZL5DCFzrOqINewOZBaAn4TpkdSfmZUA65KZe8qrOmw7YbVxX8HOP0RPPtA96 +zbUtc2VjHT0lNMZmiTuuLYtKxV+dyyAqX48KxthNCCs0zP414nUXxymdE0MCSUss +565FXf+FrMA+owe1SlacX/IjCkgN2oauRMDXN+JDLXJUwDKVKOb3yObXyIJ0/b6E ++cGmwo/7m6CE5hoCSncE +-----END CERTIFICATE----- diff --git a/tests/integration/test_reload_certificate/configs/second.key b/tests/integration/test_reload_certificate/configs/second.key new file mode 100644 index 00000000000..007aba24fb6 --- /dev/null +++ b/tests/integration/test_reload_certificate/configs/second.key @@ -0,0 +1,28 @@ +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDrQQCvvoSpNjl1 +HvPgI9ST+UbjZGEYHLVPlLTVh/Q/RIQg/zxVOWmuHOnFXRAyb2b7B8uuNBC8wzHa +ubLkP6wk8nBbhc5kK4ohhiFA3DD86DGXpxfZhlZ/x/mnQnb8T+PFSPXNfJxTer1R +ttBBzHSiRcG0cTkCPH0oIYBRbNAOIg7/76EGHhNNBLDgU7CaMpvOeefMVJ1qd5SY +RDgLRvZa4Y3KWtA9WrTQmDTH6YzH+bLnVBfUV5rs9nyM4B8pGNrezb/deFlzB9c8 ++FhFxxm8UjeOZZhzyro+7eToVpDfbtuYcmjgju7O3142/s2P29RTogCteA8PP4KH +c6oekztDAgMBAAECggEANQeZFQSYOOB9QTZx+ON6xsRZQ2bcMChAgqjdvoh/+UcD +lcCTJA7mEJZ558Bbp1LPXuTZ9/HKmBJUCZ70gVkM/+Mairb12ESsRXRLyKgZ7tiU +XUAQMzuCAhnc3+QumB+WE2Gn7uMZBgRT6riP51UkMXQR/w/KrwNdnw82MqSZnaWH +G2U+VWzrwloGDn4wdTm0egPMUCoF9kTW04HCTRZBg2a1DGCIOd2QcnYrsLiRQHmK +J8hfVVHDk+aaDmkYhJqyruTan51ifWziMOAcxTxuhDDtGOGXtMMlJOF9sskVT8CF +ToCSMJ8Rs+TPHqdBTy9l9n6kSki9Z/JFylMBClQsAQKBgQD6/FF5EF7WaVjs6gJt +DcOCPBAo8Se1YxFQ+lHIflswJs9O8u/wOej06o3c2S0zqj8v6aomhw4hYds2RaSa +fJ1jOS/7l+b3A4QndmZ6ZPBXc4mdWeUwcJ1snzi8of7D+BInAoXa9mwmOYp4u8DB +x+udumDr9zorR3aI6LXARF8+AQKBgQDv9DlBKXO7iqf2n3peb5QaLuxLvaiQM2+I +kA5xTakrODWE8+nZ0fTjrq5gVsYoIGaci6FlvX1yKsWiwy79ng+XBPNcz14FEhrn +xQFDWa/t2xEMruar3cQ0eqotlwe4yanM+/5SYu5kgm8qs9CUdr14vrV3MYB0moKb +YAg2qmgBQwKBgQDavGPU+qtsecuCTj9nA4PMUMRUqjdNIdXJmR8FePnH8UrjJ15t +Iksgh/qy6qM2T71Z6G7dvP5XoY0Gs5NNACW6f/CNeElWJb5bFhkhui6sSIk6lUnk ++YB5VhqAaz45VE2dqdk2h2Shu6wupJLNT4rMn84wV/peFZ38m7MqqWvIAQKBgCh4 +CxP3VsKBfxR0DyJQNS05TrbzdLNlSWFB0n2/eFGGuFgE/yKya1ffBR/QYrkvxb6P +Ohg7niWcGxr5SjqR5tU0i4rSmmvGgu0l57GhNa+q67Q050iDLW0gZwUrXK0Ire+Z +bGoer1AaQ39zNjFj2U6880P4AE8qI+7qglgd406bAoGAVhnd9sA+hU6pG+WJ7JbX +TQoUUmO2BF3M2C3F6dje2LNsgtuZg+YkhG4RyBFuKOCZKdN6IhfoMhBw08qskB8J +a9vFyzygqEH4X2yhgmsMpb1dHOyCIvyPwlZIeGkzzJcXlcdx6cOQvmt1NLXPwrAz +GJbH6utej4bup+u4gDT5wEk= +-----END PRIVATE KEY----- diff --git a/tests/integration/test_reload_certificate/test.py b/tests/integration/test_reload_certificate/test.py new file mode 100644 index 00000000000..dc0c391d6f0 --- /dev/null +++ b/tests/integration/test_reload_certificate/test.py @@ -0,0 +1,75 @@ +import pytest +import os +from helpers.cluster import ClickHouseCluster + +SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance('node', main_configs=["configs/first.crt", "configs/first.key", + "configs/second.crt", "configs/second.key", + "configs/cert.xml"]) + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + +def change_config_to_key(name): + ''' + * Generate config with certificate/key name from args. + * Reload config. + ''' + node.exec_in_container(["bash", "-c" , """cat > /etc/clickhouse-server/config.d/cert.xml << EOF + + + 8443 + + + /etc/clickhouse-server/config.d/{cur_name}.crt + /etc/clickhouse-server/config.d/{cur_name}.key + true + true + sslv2,sslv3 + true + + + +EOF""".format(cur_name=name)]) + node.query("SYSTEM RELOAD CONFIG") + +def test_first_than_second_cert(): + ''' Consistently set first key and check that only it will be accepted, then repeat same for second key. ''' + # Set first key + change_config_to_key('first') + + # Command with correct certificate + assert node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='first'), + 'https://localhost:8443/']) == 'Ok.\n' + + # Command with wrong certificate + # This command don't use option '-k', so it will lead to error while execution. + # That's why except will always work + try: + node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='second'), + 'https://localhost:8443/']) + assert False + except: + assert True + + # Change to other key + change_config_to_key('second') + + # Command with correct certificate + assert node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='second'), + 'https://localhost:8443/']) == 'Ok.\n' + + # Command with wrong certificate + # Same as previous + try: + node.exec_in_container(['curl', '--silent', '--cacert', '/etc/clickhouse-server/config.d/{cur_name}.crt'.format(cur_name='first'), + 'https://localhost:8443/']) + assert False + except: + assert True diff --git a/tests/integration/test_role/test.py b/tests/integration/test_role/test.py index 1e253a93737..7600bc73b16 100644 --- a/tests/integration/test_role/test.py +++ b/tests/integration/test_role/test.py @@ -225,11 +225,11 @@ def test_introspection(): ["R2", "local directory"]]) assert instance.query( - "SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, grant_option") == \ + "SELECT * from system.grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, access_type, database, table, column, is_partial_revoke, grant_option") == \ TSV([["A", "\\N", "SELECT", "test", "table", "\\N", 0, 0], ["B", "\\N", "CREATE", "\\N", "\\N", "\\N", 0, 1], - ["\\N", "R2", "SELECT", "test", "table", "\\N", 0, 0], - ["\\N", "R2", "SELECT", "test", "table", "x", 1, 0]]) + ["\\N", "R2", "SELECT", "test", "table", "x", 1, 0], + ["\\N", "R2", "SELECT", "test", "table", "\\N", 0, 0]]) assert instance.query( "SELECT * from system.role_grants WHERE user_name IN ('A', 'B') OR role_name IN ('R1', 'R2') ORDER BY user_name, role_name, granted_role_name") == \ diff --git a/tests/integration/test_row_policy/test.py b/tests/integration/test_row_policy/test.py index 66a35bea06b..0a7f6958b4a 100644 --- a/tests/integration/test_row_policy/test.py +++ b/tests/integration/test_row_policy/test.py @@ -389,6 +389,52 @@ def test_dcl_management(): assert node.query("SHOW POLICIES") == "" +def test_grant_create_row_policy(): + copy_policy_xml('no_filters.xml') + assert node.query("SHOW POLICIES") == "" + node.query("CREATE USER X") + + expected_error = "necessary to have grant CREATE ROW POLICY ON mydb.filtered_table1" + assert expected_error in node.query_and_get_error("CREATE POLICY pA ON mydb.filtered_table1 FOR SELECT USING a + + + + + + s3 + http://minio1:9001/root/data/ + minio + minio123 + + + + + + +
+ s3_disk +
+ + + + + diff --git a/tests/integration/test_s3_low_cardinality_right_border/test.py b/tests/integration/test_s3_low_cardinality_right_border/test.py new file mode 100644 index 00000000000..056c3e4430f --- /dev/null +++ b/tests/integration/test_s3_low_cardinality_right_border/test.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python3 + +# 1) Here we try to reproduce very rare error which is connected with +# LowCardinality. When we read data from S3 we are trying to make sequential +# reads without additional seeks. To achieve this we are trying to have as +# large mark ranges as possible for each thread which read data from S3. +# Additionaly, to avoid redundant reads we specify the "right border" for each +# read. Such possiblity supported by S3 API. For example you can send request +# to S3 to read data from 563 byte to 92753 byte and we use this feature in +# ClickHouse. +# +# 2) We use granules (range of data between marks) as a minimal task for each +# thread. For example, when we need to read data from 0 to 1092 mark and we +# have two threads with one task for each of them: thread_1 = [0, 546), +# thread_2 = [546, 1092). Of course S3 API knows nothing about marks, it works +# with bytes. So, each marks points to some offset in compressed file (stored +# in S3) and offset in decompressed block (here we don't need it). So to convert +# our mark range into bytes range we use range.begin_mark.offset_in_compressed_file as +# begin of bytes range and range.end.offset_in_compressed_file as end of bytes range. It +# works most of the times, because this last mark in range is not included and we can its +# offset_in_compressed_file as end for our range. +# +# LowCardinality serialization format consist of two files (except files for marks): +# file with index (column_name.bin) and file with dictionary (column_name.dict.bin). Data +# in index file points to real column values in dictionary. Also dictionary can be shared between +# several index marks (when you have a lot of rows with same value), for example: +# .... +# Mark 186, points to [2003111, 0] +# Mark 187, points to [2003111, 0] +# Mark 188, points to [2003111, 0] +# Mark 189, points to [2003111, 0] +# Mark 190, points to [2003111, 0] +# Mark 191, points to [2003111, 0] +# Mark 192, points to [2081424, 0] +# Mark 193, points to [2081424, 0] +# Mark 194, points to [2081424, 0] +# Mark 195, points to [2081424, 0] +# Mark 196, points to [2081424, 0] +# Mark 197, points to [2081424, 0] +# Mark 198, points to [2081424, 0] +# Mark 199, points to [2081424, 0] +# Mark 200, points to [2081424, 0] +# Mark 201, points to [2159750, 0] +# Mark 202, points to [2159750, 0] +# Mark 203, points to [2159750, 0] +# Mark 204, points to [2159750, 0] +# .... +# +# Imagine, this case when we have two threads: [0, 189) and [189, 378). Which +# bytes range we will have? Using logic from 2) we will get +# [0.offset_in_compressed_file, 189.offset_in_compressed_file] = [0, 2003111]. +# But it's incorrect range, because actually dictionary ends in offset 2081424, +# but all marks from 186 to 191 share this same dictionary. If we try to read +# data from [0, 2003111] we will not be able to do it, because it will be +# impossible to read dictionary. +# +# So this buggy logic was fixed and this test confirms this. At first I've +# tried to get sane numbers for data, but the error didn't reproduce. After +# three tries with almost random numbers of rows the error was reproduced. + + +import pytest +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance("node1", main_configs=["configs/s3.xml"], with_minio=True) + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + + yield cluster + finally: + cluster.shutdown() + + +def test_s3_right_border(started_cluster): + node1.query(""" +CREATE TABLE s3_low_cardinality +( + str_column LowCardinality(String) +) +ENGINE = MergeTree() +ORDER BY tuple() +SETTINGS storage_policy = 's3', min_bytes_for_wide_part = 0, index_granularity = 1024; + """) + + node1.query("INSERT INTO s3_low_cardinality SELECT 'aaaaaa' FROM numbers(600000)") + node1.query("INSERT INTO s3_low_cardinality SELECT toString(number) FROM numbers(100000)") + node1.query("INSERT INTO s3_low_cardinality SELECT 'bbbbbb' FROM numbers(500000)") + node1.query("INSERT INTO s3_low_cardinality SELECT toString(number + 100000000) FROM numbers(100000)") + + node1.query("OPTIMIZE TABLE s3_low_cardinality FINAL") + + settings = { + "merge_tree_min_bytes_for_concurrent_read": "0", + "merge_tree_min_rows_for_concurrent_read": "0", + "max_threads": "2", + } + assert node1.query("SELECT COUNT() FROM s3_low_cardinality WHERE not ignore(str_column)", settings=settings) == "1300000\n" diff --git a/tests/integration/test_s3_zero_copy_replication/test.py b/tests/integration/test_s3_zero_copy_replication/test.py index fb30a83877b..c56d98559f8 100644 --- a/tests/integration/test_s3_zero_copy_replication/test.py +++ b/tests/integration/test_s3_zero_copy_replication/test.py @@ -329,6 +329,7 @@ def test_s3_zero_copy_unfreeze(cluster): check_objects_exisis(cluster, objects01) node1.query("TRUNCATE TABLE unfreeze_test") + node2.query("SYSTEM SYNC REPLICA unfreeze_test") objects11 = node1.get_backuped_s3_objects("s31", "freeze_backup1") objects12 = node2.get_backuped_s3_objects("s31", "freeze_backup2") @@ -373,6 +374,7 @@ def test_s3_zero_copy_drop_detached(cluster): node1.query("ALTER TABLE drop_detached_test FREEZE WITH NAME 'detach_backup1'") node1.query("INSERT INTO drop_detached_test VALUES (1)") node1.query("ALTER TABLE drop_detached_test FREEZE WITH NAME 'detach_backup2'") + node2.query("SYSTEM SYNC REPLICA drop_detached_test") objects1 = node1.get_backuped_s3_objects("s31", "detach_backup1") objects2 = node1.get_backuped_s3_objects("s31", "detach_backup2") @@ -384,6 +386,8 @@ def test_s3_zero_copy_drop_detached(cluster): node1.query("ALTER TABLE drop_detached_test DETACH PARTITION '0'") node1.query("ALTER TABLE drop_detached_test DETACH PARTITION '1'") + node2.query("SYSTEM SYNC REPLICA drop_detached_test") + wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) @@ -391,6 +395,7 @@ def test_s3_zero_copy_drop_detached(cluster): check_objects_exisis(cluster, objects2) node2.query("ALTER TABLE drop_detached_test DROP DETACHED PARTITION '1'", settings={"allow_drop_detached": 1}) + node1.query("SYSTEM SYNC REPLICA drop_detached_test") wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) @@ -398,6 +403,7 @@ def test_s3_zero_copy_drop_detached(cluster): check_objects_exisis(cluster, objects2) node1.query("ALTER TABLE drop_detached_test DROP DETACHED PARTITION '1'", settings={"allow_drop_detached": 1}) + node2.query("SYSTEM SYNC REPLICA drop_detached_test") wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) @@ -405,12 +411,14 @@ def test_s3_zero_copy_drop_detached(cluster): check_objects_not_exisis(cluster, objects_diff) node1.query("ALTER TABLE drop_detached_test DROP DETACHED PARTITION '0'", settings={"allow_drop_detached": 1}) + node2.query("SYSTEM SYNC REPLICA drop_detached_test") wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) check_objects_exisis(cluster, objects1) node2.query("ALTER TABLE drop_detached_test DROP DETACHED PARTITION '0'", settings={"allow_drop_detached": 1}) + node1.query("SYSTEM SYNC REPLICA drop_detached_test") wait_mutations(node1, "drop_detached_test", 10) wait_mutations(node2, "drop_detached_test", 10) diff --git a/tests/integration/test_server_reload/protos/clickhouse_grpc.proto b/tests/integration/test_server_reload/protos/clickhouse_grpc.proto deleted file mode 100644 index c6cafaf6e40..00000000000 --- a/tests/integration/test_server_reload/protos/clickhouse_grpc.proto +++ /dev/null @@ -1,174 +0,0 @@ -/* This file describes gRPC protocol supported in ClickHouse. - * - * To use this protocol a client should send one or more messages of the QueryInfo type - * and then receive one or more messages of the Result type. - * According to that the service provides four methods for that: - * ExecuteQuery(QueryInfo) returns (Result) - * ExecuteQueryWithStreamInput(stream QueryInfo) returns (Result) - * ExecuteQueryWithStreamOutput(QueryInfo) returns (stream Result) - * ExecuteQueryWithStreamIO(stream QueryInfo) returns (stream Result) - * It's up to the client to choose which method to use. - * For example, ExecuteQueryWithStreamInput() allows the client to add data multiple times - * while executing a query, which is suitable for inserting many rows. - */ - -syntax = "proto3"; - -package clickhouse.grpc; - -message NameAndType { - string name = 1; - string type = 2; -} - -// Describes an external table - a table which will exists only while a query is executing. -message ExternalTable { - // Name of the table. If omitted, "_data" is used. - string name = 1; - - // Columns of the table. Types are required, names can be omitted. If the names are omitted, "_1", "_2", ... is used. - repeated NameAndType columns = 2; - - // Data to insert to the external table. - // If a method with streaming input (i.e. ExecuteQueryWithStreamInput() or ExecuteQueryWithStreamIO()) is used, - // then data for insertion to the same external table can be split between multiple QueryInfos. - bytes data = 3; - - // Format of the data to insert to the external table. - string format = 4; - - // Settings for executing that insertion, applied after QueryInfo.settings. - map settings = 5; -} - -enum CompressionAlgorithm { - NO_COMPRESSION = 0; - DEFLATE = 1; - GZIP = 2; - STREAM_GZIP = 3; -} - -enum CompressionLevel { - COMPRESSION_NONE = 0; - COMPRESSION_LOW = 1; - COMPRESSION_MEDIUM = 2; - COMPRESSION_HIGH = 3; -} - -message Compression { - CompressionAlgorithm algorithm = 1; - CompressionLevel level = 2; -} - -// Information about a query which a client sends to a ClickHouse server. -// The first QueryInfo can set any of the following fields. Extra QueryInfos only add extra data. -// In extra QueryInfos only `input_data`, `external_tables`, `next_query_info` and `cancel` fields can be set. -message QueryInfo { - string query = 1; - string query_id = 2; - map settings = 3; - - // Default database. - string database = 4; - - // Input data, used both as data for INSERT query and as data for the input() function. - bytes input_data = 5; - - // Delimiter for input_data, inserted between input_data from adjacent QueryInfos. - bytes input_data_delimiter = 6; - - // Default output format. If not specified, 'TabSeparated' is used. - string output_format = 7; - - repeated ExternalTable external_tables = 8; - - string user_name = 9; - string password = 10; - string quota = 11; - - // Works exactly like sessions in the HTTP protocol. - string session_id = 12; - bool session_check = 13; - uint32 session_timeout = 14; - - // Set `cancel` to true to stop executing the query. - bool cancel = 15; - - // If true there will be at least one more QueryInfo in the input stream. - // `next_query_info` is allowed to be set only if a method with streaming input (i.e. ExecuteQueryWithStreamInput() or ExecuteQueryWithStreamIO()) is used. - bool next_query_info = 16; - - /// Controls how a ClickHouse server will compress query execution results before sending back to the client. - /// If not set the compression settings from the configuration file will be used. - Compression result_compression = 17; -} - -enum LogsLevel { - LOG_NONE = 0; - LOG_FATAL = 1; - LOG_CRITICAL = 2; - LOG_ERROR = 3; - LOG_WARNING = 4; - LOG_NOTICE = 5; - LOG_INFORMATION = 6; - LOG_DEBUG = 7; - LOG_TRACE = 8; -} - -message LogEntry { - uint32 time = 1; - uint32 time_microseconds = 2; - uint64 thread_id = 3; - string query_id = 4; - LogsLevel level = 5; - string source = 6; - string text = 7; -} - -message Progress { - uint64 read_rows = 1; - uint64 read_bytes = 2; - uint64 total_rows_to_read = 3; - uint64 written_rows = 4; - uint64 written_bytes = 5; -} - -message Stats { - uint64 rows = 1; - uint64 blocks = 2; - uint64 allocated_bytes = 3; - bool applied_limit = 4; - uint64 rows_before_limit = 5; -} - -message Exception { - int32 code = 1; - string name = 2; - string display_text = 3; - string stack_trace = 4; -} - -// Result of execution of a query which is sent back by the ClickHouse server to the client. -message Result { - // Output of the query, represented in the `output_format` or in a format specified in `query`. - bytes output = 1; - bytes totals = 2; - bytes extremes = 3; - - repeated LogEntry logs = 4; - Progress progress = 5; - Stats stats = 6; - - // Set by the ClickHouse server if there was an exception thrown while executing. - Exception exception = 7; - - // Set by the ClickHouse server if executing was cancelled by the `cancel` field in QueryInfo. - bool cancelled = 8; -} - -service ClickHouse { - rpc ExecuteQuery(QueryInfo) returns (Result) {} - rpc ExecuteQueryWithStreamInput(stream QueryInfo) returns (Result) {} - rpc ExecuteQueryWithStreamOutput(QueryInfo) returns (stream Result) {} - rpc ExecuteQueryWithStreamIO(stream QueryInfo) returns (stream Result) {} -} diff --git a/tests/integration/test_server_reload/protos/clickhouse_grpc.proto b/tests/integration/test_server_reload/protos/clickhouse_grpc.proto new file mode 120000 index 00000000000..25d15f11e3b --- /dev/null +++ b/tests/integration/test_server_reload/protos/clickhouse_grpc.proto @@ -0,0 +1 @@ +../../../../src/Server/grpc_protos/clickhouse_grpc.proto \ No newline at end of file diff --git a/tests/integration/test_storage_kafka/test.py b/tests/integration/test_storage_kafka/test.py index a92dafa0b8a..8326797f96d 100644 --- a/tests/integration/test_storage_kafka/test.py +++ b/tests/integration/test_storage_kafka/test.py @@ -1572,7 +1572,7 @@ def test_kafka_virtual_columns_with_materialized_view(kafka_cluster): messages.append(json.dumps({'key': i, 'value': i})) kafka_produce(kafka_cluster, 'virt2', messages, 0) - sql = 'SELECT kafka_key, key, topic, value, offset, partition, timestamp FROM test.view ORDER BY kafka_key' + sql = 'SELECT kafka_key, key, topic, value, offset, partition, timestamp FROM test.view ORDER BY kafka_key, key' result = instance.query(sql) iterations = 0 while not kafka_check_result(result, False, 'test_kafka_virtual2.reference') and iterations < 10: diff --git a/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh index 0a746eb1a67..45fb93792e0 100644 --- a/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh +++ b/tests/integration/test_storage_kerberized_hdfs/kerberos_image_config.sh @@ -90,6 +90,7 @@ create_admin_user() { } create_keytabs() { + rm /tmp/keytab/*.keytab # kadmin.local -q "addprinc -randkey hdfs/kerberizedhdfs1.${DOMAIN_REALM}@${REALM}" diff --git a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh index 723868ec68a..07437c42359 100644 --- a/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh +++ b/tests/integration/test_storage_kerberized_kafka/kerberos_image_config.sh @@ -90,6 +90,7 @@ create_admin_user() { } create_keytabs() { + rm /tmp/keytab/*.keytab kadmin.local -q "addprinc -randkey zookeeper/kafka_kerberized_zookeeper@${REALM}" kadmin.local -q "ktadd -norandkey -k /tmp/keytab/kafka_kerberized_zookeeper.keytab zookeeper/kafka_kerberized_zookeeper@${REALM}" diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index b6ac121cd0c..87337a6b459 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -13,11 +13,19 @@ node2 = cluster.add_instance('node2', main_configs=['configs/named_collections.x def started_cluster(): try: cluster.start() + node1.query("CREATE DATABASE test") yield cluster finally: cluster.shutdown() +@pytest.fixture(autouse=True) +def setup_teardown(): + print("PostgreSQL is available - running test") + yield # run test + node1.query("DROP DATABASE test") + node1.query("CREATE DATABASE test") + def test_postgres_select_insert(started_cluster): cursor = started_cluster.postgres_conn.cursor() table_name = 'test_many' @@ -143,11 +151,11 @@ def test_non_default_scema(started_cluster): cursor.execute('INSERT INTO test_schema.test_table SELECT i FROM generate_series(0, 99) as t(i)') node1.query(''' - CREATE TABLE test_pg_table_schema (a UInt32) + CREATE TABLE test.test_pg_table_schema (a UInt32) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_table', 'postgres', 'mysecretpassword', 'test_schema'); ''') - result = node1.query('SELECT * FROM test_pg_table_schema') + result = node1.query('SELECT * FROM test.test_pg_table_schema') expected = node1.query('SELECT number FROM numbers(100)') assert(result == expected) @@ -160,10 +168,10 @@ def test_non_default_scema(started_cluster): cursor.execute('INSERT INTO "test.nice.schema"."test.nice.table" SELECT i FROM generate_series(0, 99) as t(i)') node1.query(''' - CREATE TABLE test_pg_table_schema_with_dots (a UInt32) + CREATE TABLE test.test_pg_table_schema_with_dots (a UInt32) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test.nice.table', 'postgres', 'mysecretpassword', 'test.nice.schema'); ''') - result = node1.query('SELECT * FROM test_pg_table_schema_with_dots') + result = node1.query('SELECT * FROM test.test_pg_table_schema_with_dots') assert(result == expected) cursor.execute('INSERT INTO "test_schema"."test_table" SELECT i FROM generate_series(100, 199) as t(i)') @@ -173,8 +181,8 @@ def test_non_default_scema(started_cluster): cursor.execute('DROP SCHEMA test_schema CASCADE') cursor.execute('DROP SCHEMA "test.nice.schema" CASCADE') - node1.query('DROP TABLE test_pg_table_schema') - node1.query('DROP TABLE test_pg_table_schema_with_dots') + node1.query('DROP TABLE test.test_pg_table_schema') + node1.query('DROP TABLE test.test_pg_table_schema_with_dots') def test_concurrent_queries(started_cluster): @@ -302,19 +310,19 @@ def test_postgres_distributed(started_cluster): def test_datetime_with_timezone(started_cluster): cursor = started_cluster.postgres_conn.cursor() cursor.execute("DROP TABLE IF EXISTS test_timezone") - node1.query("DROP TABLE IF EXISTS test_timezone") + node1.query("DROP TABLE IF EXISTS test.test_timezone") cursor.execute("CREATE TABLE test_timezone (ts timestamp without time zone, ts_z timestamp with time zone)") cursor.execute("insert into test_timezone select '2014-04-04 20:00:00', '2014-04-04 20:00:00'::timestamptz at time zone 'America/New_York';") cursor.execute("select * from test_timezone") result = cursor.fetchall()[0] logging.debug(f'{result[0]}, {str(result[1])[:-6]}') - node1.query("create table test_timezone ( ts DateTime, ts_z DateTime('America/New_York')) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_timezone', 'postgres', 'mysecretpassword');") - assert(node1.query("select ts from test_timezone").strip() == str(result[0])) + node1.query("create table test.test_timezone ( ts DateTime, ts_z DateTime('America/New_York')) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_timezone', 'postgres', 'mysecretpassword');") + assert(node1.query("select ts from test.test_timezone").strip() == str(result[0])) # [:-6] because 2014-04-04 16:00:00+00:00 -> 2014-04-04 16:00:00 - assert(node1.query("select ts_z from test_timezone").strip() == str(result[1])[:-6]) - assert(node1.query("select * from test_timezone") == "2014-04-04 20:00:00\t2014-04-04 16:00:00\n") + assert(node1.query("select ts_z from test.test_timezone").strip() == str(result[1])[:-6]) + assert(node1.query("select * from test.test_timezone") == "2014-04-04 20:00:00\t2014-04-04 16:00:00\n") cursor.execute("DROP TABLE test_timezone") - node1.query("DROP TABLE test_timezone") + node1.query("DROP TABLE test.test_timezone") def test_postgres_ndim(started_cluster): @@ -342,20 +350,20 @@ def test_postgres_on_conflict(started_cluster): cursor.execute(f'CREATE TABLE {table} (a integer PRIMARY KEY, b text, c integer)') node1.query(''' - CREATE TABLE test_conflict (a UInt32, b String, c Int32) + CREATE TABLE test.test_conflict (a UInt32, b String, c Int32) ENGINE PostgreSQL('postgres1:5432', 'postgres', 'test_conflict', 'postgres', 'mysecretpassword', '', 'ON CONFLICT DO NOTHING'); ''') - node1.query(f''' INSERT INTO {table} SELECT number, concat('name_', toString(number)), 3 from numbers(100)''') - node1.query(f''' INSERT INTO {table} SELECT number, concat('name_', toString(number)), 4 from numbers(100)''') + node1.query(f''' INSERT INTO test.{table} SELECT number, concat('name_', toString(number)), 3 from numbers(100)''') + node1.query(f''' INSERT INTO test.{table} SELECT number, concat('name_', toString(number)), 4 from numbers(100)''') - check1 = f"SELECT count() FROM {table}" + check1 = f"SELECT count() FROM test.{table}" assert (node1.query(check1)).rstrip() == '100' table_func = f'''postgresql('{started_cluster.postgres_ip}:{started_cluster.postgres_port}', 'postgres', '{table}', 'postgres', 'mysecretpassword', '', 'ON CONFLICT DO NOTHING')''' node1.query(f'''INSERT INTO TABLE FUNCTION {table_func} SELECT number, concat('name_', toString(number)), 3 from numbers(100)''') node1.query(f'''INSERT INTO TABLE FUNCTION {table_func} SELECT number, concat('name_', toString(number)), 3 from numbers(100)''') - check1 = f"SELECT count() FROM {table}" + check1 = f"SELECT count() FROM test.{table}" assert (node1.query(check1)).rstrip() == '100' cursor.execute(f'DROP TABLE {table} ') @@ -367,48 +375,48 @@ def test_predefined_connection_configuration(started_cluster): cursor.execute(f'CREATE TABLE test_table (a integer PRIMARY KEY, b integer)') node1.query(''' - DROP TABLE IF EXISTS test_table; - CREATE TABLE test_table (a UInt32, b Int32) + DROP TABLE IF EXISTS test.test_table; + CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres1); ''') - node1.query(f''' INSERT INTO test_table SELECT number, number from numbers(100)''') - assert (node1.query(f"SELECT count() FROM test_table").rstrip() == '100') + node1.query(f''' INSERT INTO test.test_table SELECT number, number from numbers(100)''') + assert (node1.query(f"SELECT count() FROM test.test_table").rstrip() == '100') node1.query(''' - DROP TABLE test_table; - CREATE TABLE test_table (a UInt32, b Int32) + DROP TABLE test.test_table; + CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres1, on_conflict='ON CONFLICT DO NOTHING'); ''') - node1.query(f''' INSERT INTO test_table SELECT number, number from numbers(100)''') - node1.query(f''' INSERT INTO test_table SELECT number, number from numbers(100)''') - assert (node1.query(f"SELECT count() FROM test_table").rstrip() == '100') + node1.query(f''' INSERT INTO test.test_table SELECT number, number from numbers(100)''') + node1.query(f''' INSERT INTO test.test_table SELECT number, number from numbers(100)''') + assert (node1.query(f"SELECT count() FROM test.test_table").rstrip() == '100') - node1.query('DROP TABLE test_table;') + node1.query('DROP TABLE test.test_table;') node1.query_and_get_error(''' - CREATE TABLE test_table (a UInt32, b Int32) + CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres1, 'ON CONFLICT DO NOTHING'); ''') node1.query_and_get_error(''' - CREATE TABLE test_table (a UInt32, b Int32) + CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres2); ''') node1.query_and_get_error(''' - CREATE TABLE test_table (a UInt32, b Int32) + CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(unknown_collection); ''') node1.query(''' - CREATE TABLE test_table (a UInt32, b Int32) + CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres1, port=5432, database='postgres', table='test_table'); ''') - assert (node1.query(f"SELECT count() FROM test_table").rstrip() == '100') + assert (node1.query(f"SELECT count() FROM test.test_table").rstrip() == '100') node1.query(''' - DROP TABLE test_table; - CREATE TABLE test_table (a UInt32, b Int32) + DROP TABLE test.test_table; + CREATE TABLE test.test_table (a UInt32, b Int32) ENGINE PostgreSQL(postgres3, port=5432); ''') - assert (node1.query(f"SELECT count() FROM test_table").rstrip() == '100') + assert (node1.query(f"SELECT count() FROM test.test_table").rstrip() == '100') assert (node1.query(f"SELECT count() FROM postgresql(postgres1)").rstrip() == '100') node1.query("INSERT INTO TABLE FUNCTION postgresql(postgres1, on_conflict='ON CONFLICT DO NOTHING') SELECT number, number from numbers(100)") diff --git a/tests/integration/test_storage_s3/configs/named_collections.xml b/tests/integration/test_storage_s3/configs/named_collections.xml index f22440d17c9..fcc8bcac555 100644 --- a/tests/integration/test_storage_s3/configs/named_collections.xml +++ b/tests/integration/test_storage_s3/configs/named_collections.xml @@ -30,5 +30,10 @@ minio minio123 + + http://minio1:9001/root/test.parquet + minio + minio123 + diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index a804053d4fd..3fc44469998 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -901,6 +901,13 @@ def test_s3_schema_inference(started_cluster): result = instance.query(f"select count(*) from schema_inference_2") assert(int(result) == 5000000) + table_function = f"s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test_native', 'Native')" + result = instance.query(f"desc {table_function}") + assert result == "a\tInt32\t\t\t\t\t\nb\tString\t\t\t\t\t\n" + + result = instance.query(f"select count(*) from {table_function}") + assert(int(result) == 5000000) + def test_empty_file(started_cluster): bucket = started_cluster.minio_bucket @@ -971,3 +978,42 @@ def test_format_detection(started_cluster): result = instance.query(f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')") assert(int(result) == 1) + result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')") + assert(int(result) == 1) + + + instance.query(f"create table parquet_table_s3 (x UInt64) engine=S3(s3_parquet2)") + instance.query(f"insert into parquet_table_s3 select 1") + result = instance.query(f"select * from s3(s3_parquet2)") + assert(int(result) == 1) + + result = instance.query(f"select * from url('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.parquet')") + assert(int(result) == 1) + + result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.parquet')") + assert(int(result) == 1) + + +def test_signatures(started_cluster): + bucket = started_cluster.minio_bucket + instance = started_cluster.instances["dummy"] + + instance.query(f"create table test_signatures (x UInt64) engine=S3(s3_arrow)") + instance.query(f"truncate table test_signatures") + instance.query(f"insert into test_signatures select 1") + + result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow')") + assert(int(result) == 1) + + result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'Arrow', 'x UInt64')") + assert(int(result) == 1) + + result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'minio', 'minio123')") + assert(int(result) == 1) + + result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'Arrow', 'x UInt64', 'auto')") + assert(int(result) == 1) + + result = instance.query(f"select * from s3('http://{started_cluster.minio_host}:{started_cluster.minio_port}/{bucket}/test.arrow', 'minio', 'minio123', 'Arrow')") + assert(int(result) == 1) + diff --git a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml index a76e984e4e6..ae1dc9dd038 100644 --- a/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml +++ b/tests/integration/test_ttl_move/configs/config.d/storage_configuration.xml @@ -76,6 +76,14 @@ + + +
+ jbod1 +
+ + +
diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 254447478f9..d8373ccb48a 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1174,3 +1174,57 @@ def test_disabled_ttl_move_on_insert(started_cluster, name, dest_type, engine): node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) except: pass + + +@pytest.mark.parametrize("name,dest_type", [ + pytest.param("replicated_mt_move_if_exists", "DISK", id="replicated_disk"), + pytest.param("replicated_mt_move_if_exists", "VOLUME", id="replicated_volume"), +]) +def test_ttl_move_if_exists(started_cluster, name, dest_type): + name = unique_table_name(name) + + try: + query_template = """ + CREATE TABLE {name} ( + s1 String, + d1 DateTime + ) ENGINE = ReplicatedMergeTree('/clickhouse/replicated_mt_move_if_exists', '{node_name}') + ORDER BY tuple() + TTL d1 TO {dest_type} {if_exists} 'external' + SETTINGS storage_policy='{policy}' + """ + + with pytest.raises(QueryRuntimeException): + node1.query(query_template.format( \ + name=name, node_name=node1.name, dest_type=dest_type, \ + if_exists='', policy='only_jbod_1')) + + for (node, policy) in zip([node1, node2], ['only_jbod_1', 'small_jbod_with_external']): + node.query(query_template.format( \ + name=name, node_name=node.name, dest_type=dest_type, \ + if_exists='IF EXISTS', policy=policy)) + + data = [] # 10MB in total + for i in range(10): + data.append(("randomPrintableASCII(1024*1024)", "toDateTime({})".format(time.time() - 1))) + + node1.query("INSERT INTO {} (s1, d1) VALUES {}".format(name, ",".join(["(" + ",".join(x) + ")" for x in data]))) + node2.query("SYSTEM SYNC REPLICA {}".format(name)) + + time.sleep(5) + + used_disks1 = get_used_disks_for_table(node1, name) + assert set(used_disks1) == {"jbod1"} + + used_disks2 = get_used_disks_for_table(node2, name) + assert set(used_disks2) == {"external"} + + assert node1.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + assert node2.query("SELECT count() FROM {name}".format(name=name)).strip() == "10" + + finally: + try: + node1.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + node2.query("DROP TABLE IF EXISTS {} NO DELAY".format(name)) + except: + pass diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml index 7078cb16002..ec88be0124f 100644 --- a/tests/performance/decimal_aggregates.xml +++ b/tests/performance/decimal_aggregates.xml @@ -28,6 +28,11 @@ SELECT quantile(d64), quantileExact(d64), quantileExactWeighted(d64, 2) FROM (SELECT * FROM t LIMIT 1000000) SELECT quantile(d128), quantileExact(d128), quantileExactWeighted(d128, 2) FROM (SELECT * FROM t LIMIT 1000000) + SELECT quantilesExactLow(0.5)(d32) FROM (SELECT * FROM t LIMIT 10000000) + SELECT quantilesExactHigh(0.5)(d32) FROM (SELECT * FROM t LIMIT 10000000) + SELECT quantilesExactLow(0.1, 0.5, 0.9)(d32) FROM (SELECT * FROM t LIMIT 10000000) + SELECT quantilesExactHigh(0.1, 0.5, 0.9)(d32) FROM (SELECT * FROM t LIMIT 10000000) + SELECT quantilesExact(0.1, 0.9)(d32), quantilesExactWeighted(0.1, 0.9)(d32, 2) FROM (SELECT * FROM t LIMIT 10000000) SELECT quantilesExact(0.1, 0.9)(d64), quantilesExactWeighted(0.1, 0.9)(d64, 2) FROM (SELECT * FROM t LIMIT 1000000) SELECT quantilesExact(0.1, 0.9)(d128), quantilesExactWeighted(0.1, 0.9)(d128, 2) FROM (SELECT * FROM t LIMIT 1000000) diff --git a/tests/performance/file_table_function.xml b/tests/performance/file_table_function.xml new file mode 100644 index 00000000000..143f2b5eb4d --- /dev/null +++ b/tests/performance/file_table_function.xml @@ -0,0 +1,54 @@ + + + + + format + + TabSeparated + TabSeparatedWithNames + TabSeparatedWithNamesAndTypes + CSV + CSVWithNames + Values + JSONEachRow + JSONCompactEachRow + JSONCompactEachRowWithNamesAndTypes + TSKV + RowBinary + Native + MsgPack + + + + partitions_count + + 5 + 50 + 500 + + + + + + INSERT INTO FUNCTION file('test_file', '{format}', 'key UInt64, value UInt64') + SELECT number, number FROM numbers(1000000) + + + + INSERT INTO FUNCTION file('test_file', '{format}', 'key UInt64, value1 UInt64, value2 UInt64, value3 UInt64, value4 UInt64, value5 UInt64') + SELECT number, number, number, number, number, number FROM numbers(1000000) + + + + INSERT INTO FUNCTION file('test_file_{{_partition_id}}', '{format}', 'partition_id UInt64, value UInt64') + PARTITION BY partition_id + SELECT (number % {partitions_count}) as partition_id, number FROM numbers(1000000) + + + + INSERT INTO FUNCTION file('test_file_{{_partition_id}}', '{format}', 'partition_id UInt64, value1 UInt64, value2 UInt64, value3 UInt64, value4 UInt64, value5 UInt64') + PARTITION BY partition_id + SELECT (number % {partitions_count}) as partition_id, number, number, number, number, number FROM numbers(1000000) + + + diff --git a/tests/performance/map_populate_series.xml b/tests/performance/map_populate_series.xml new file mode 100644 index 00000000000..db40cf09455 --- /dev/null +++ b/tests/performance/map_populate_series.xml @@ -0,0 +1,6 @@ + + SELECT mapPopulateSeries(range(number), range(number)) FROM numbers(5000) FORMAT Null; + SELECT mapPopulateSeries(range(number), range(number), 2500) FROM numbers(5000) FORMAT Null; + SELECT mapPopulateSeries(map(0, 0, number, 5)) FROM numbers(5000) FORMAT Null; + SELECT mapPopulateSeries(map(0, 0, number, 5), 2500) FROM numbers(5000) FORMAT Null; + diff --git a/tests/performance/number_formatting_formats.xml b/tests/performance/number_formatting_formats.xml index 92e04a62024..a032d546a27 100644 --- a/tests/performance/number_formatting_formats.xml +++ b/tests/performance/number_formatting_formats.xml @@ -32,7 +32,7 @@ CREATE TABLE IF NOT EXISTS table_{format} (x UInt64) ENGINE = File(`{format}`) CREATE TABLE IF NOT EXISTS table_{format_fast} (x UInt64) ENGINE = File(`{format}`) - INSERT INTO table_{format} SELECT number FROM numbers(10000000) + INSERT INTO table_{format} SELECT number FROM numbers(10000000) SETTINGS engine_file_truncate_on_insert = 1 INSERT INTO table_{format_fast} SELECT number FROM numbers(20000000) DROP TABLE IF EXISTS table_{format} diff --git a/tests/performance/order_by_tuple.xml b/tests/performance/order_by_tuple.xml new file mode 100644 index 00000000000..72fb1812bbc --- /dev/null +++ b/tests/performance/order_by_tuple.xml @@ -0,0 +1,8 @@ + + + sorting + comparison + + + select * from numbers(300000000) order by (1 - number , number + 1 , number) limit 10; + diff --git a/tests/performance/sparse_column.xml b/tests/performance/sparse_column.xml index 6523d37df44..1d270165c68 100644 --- a/tests/performance/sparse_column.xml +++ b/tests/performance/sparse_column.xml @@ -25,7 +25,7 @@ CREATE TABLE test_sparse_{ratio} (id UInt64, u8 UInt8, u64 UInt64, str String) ENGINE = MergeTree ORDER BY id - SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9 + SETTINGS ratio_of_defaults_for_sparse_serialization = 0.8 SYSTEM STOP MERGES test_{serialization}_{ratio} @@ -54,5 +54,8 @@ SELECT sum(u64) FROM test_{serialization}_{ratio} GROUP BY id % 11 FORMAT Null SELECT uniq(str) FROM test_{serialization}_{ratio} GROUP BY id % 11 FORMAT Null - + SELECT count() FROM test_{serialization}_{ratio} WHERE u64 > 0 + SELECT count() FROM test_{serialization}_{ratio} WHERE notEmpty(str) + + DROP TABLE IF EXISTS test_{serialization}_{ratio} diff --git a/tests/queries/0_stateless/00161_rounding_functions.reference b/tests/queries/0_stateless/00161_rounding_functions.reference index f0b1bc3f8ab..cca2ed3e0f7 100644 --- a/tests/queries/0_stateless/00161_rounding_functions.reference +++ b/tests/queries/0_stateless/00161_rounding_functions.reference @@ -682,3 +682,5 @@ 12345.6789 12340 12300 12000 10000 0 12345.6 12345.67 12345.678 12345.6789 12345.6789 64 64 2 0 0 0.5 0 -0.5 -0.5 -0.125 +2 20 200 5 50 500 5 50 500 +2 20 200 5 50 500 5 50 500 diff --git a/tests/queries/0_stateless/00161_rounding_functions.sql b/tests/queries/0_stateless/00161_rounding_functions.sql index cc3542338bb..abdc1e7317b 100644 --- a/tests/queries/0_stateless/00161_rounding_functions.sql +++ b/tests/queries/0_stateless/00161_rounding_functions.sql @@ -44,4 +44,7 @@ SELECT 12345.6789 AS x, floor(x, -1), floor(x, -2), floor(x, -3), floor(x, -4), SELECT roundToExp2(100), roundToExp2(64), roundToExp2(3), roundToExp2(0), roundToExp2(-1); SELECT roundToExp2(0.9), roundToExp2(0), roundToExp2(-0.5), roundToExp2(-0.6), roundToExp2(-0.2); +select round(2, 4) round2, round(20, 4) round20, round(200, 4) round200, round(5, 4) round5, round(50, 4) round50, round(500, 4) round500, round(toInt32(5), 4) roundInt5, round(toInt32(50), 4) roundInt50, round(toInt32(500), 4) roundInt500; +select roundBankers(2, 4) round2, roundBankers(20, 4) round20, roundBankers(200, 4) round200, roundBankers(5, 4) round5, roundBankers(50, 4) round50, roundBankers(500, 4) round500, roundBankers(toInt32(5), 4) roundInt5, roundBankers(toInt32(50), 4) roundInt50, roundBankers(toInt32(500), 4) roundInt500; + SELECT ceil(29375422, -54212) --{serverError 69} diff --git a/tests/queries/0_stateless/00301_csv.sh b/tests/queries/0_stateless/00301_csv.sh index e10e98a123d..50c64b312a7 100755 --- a/tests/queries/0_stateless/00301_csv.sh +++ b/tests/queries/0_stateless/00301_csv.sh @@ -15,7 +15,7 @@ Hello "world", 789 ,2016-01-03 default,, default-eof,,' | $CLICKHOUSE_CLIENT --input_format_defaults_for_omitted_fields=1 --input_format_csv_empty_as_default=1 --query="INSERT INTO csv FORMAT CSV"; -$CLICKHOUSE_CLIENT --query="SELECT * FROM csv ORDER BY d"; +$CLICKHOUSE_CLIENT --query="SELECT * FROM csv ORDER BY d, s"; $CLICKHOUSE_CLIENT --query="DROP TABLE csv"; $CLICKHOUSE_CLIENT --query="CREATE TABLE csv (t DateTime('Europe/Moscow'), s String) ENGINE = Memory"; diff --git a/tests/queries/0_stateless/00327_summing_composite_nested.reference b/tests/queries/0_stateless/00327_summing_composite_nested.reference index b4233202447..38c96d85524 100644 --- a/tests/queries/0_stateless/00327_summing_composite_nested.reference +++ b/tests/queries/0_stateless/00327_summing_composite_nested.reference @@ -1,8 +1,8 @@ 2000-01-01 1 [1,2] [3,4] [10,11] [0,1,2] ['3','4','5'] [-1,-2,-3] [1,10,100] 2000-01-01 1 [2,1] [4,3] [20,22] [2,2,1] ['5','5','0'] [-3,-3,-33] [10,100,1000] 2000-01-01 2 [1,2] [3,4] [10,11] [0,1,2] ['3','4','5'] [-1,-2,-3] [1,10,100] -2000-01-01 2 [2,1,1] [4,3,3] [20,22,33] [2,2] ['5','5'] [-3,-3] [10,100] 2000-01-01 2 [1,2] [3,4] [10,11] [0,1,2] ['3','4','5'] [-1,-2,-3] [1,10,100] +2000-01-01 2 [2,1,1] [4,3,3] [20,22,33] [2,2] ['5','5'] [-3,-3] [10,100] 2000-01-01 1 1 3 10 2000-01-01 1 1 3 22 2000-01-01 1 2 4 11 diff --git a/tests/queries/0_stateless/00327_summing_composite_nested.sql b/tests/queries/0_stateless/00327_summing_composite_nested.sql index 9be21e87abf..f9b251ebd8f 100644 --- a/tests/queries/0_stateless/00327_summing_composite_nested.sql +++ b/tests/queries/0_stateless/00327_summing_composite_nested.sql @@ -5,11 +5,11 @@ CREATE TABLE summing_composite_key (d Date, k UInt64, FirstMap Nested(k1 UInt32, INSERT INTO summing_composite_key VALUES ('2000-01-01', 1, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]), ('2000-01-01', 1, [2,1], ['4','3'], [20,22], [2,2,1], [5,5,0], [-3,-3,-33], [10,100,1000]), ('2000-01-01', 2, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]), ('2000-01-01', 2, [2,1,1], ['4','3','3'], [20,22,33], [2,2], [5,5], [-3,-3], [10,100]), ('2000-01-01', 2, [1,2], ['3','4'], [10,11], [0,1,2], [3,4,5], [-1,-2,-3], [1,10,100]); -SELECT * FROM summing_composite_key ORDER BY d, k, _part_index; +SELECT * FROM summing_composite_key ORDER BY d, k, FirstMap.k1, FirstMap.k2ID, FirstMap.s, SecondMap.k1ID, SecondMap.k2Key, SecondMap.k3Type, SecondMap.s; -SELECT d, k, m.k1, m.k2ID, m.s FROM summing_composite_key ARRAY JOIN FirstMap AS m ORDER BY d, k, m.k1, m.k2ID, m.s; +SELECT d, k, m.k1, m.k2ID, m.s FROM summing_composite_key ARRAY JOIN FirstMap AS m ORDER BY d, k, m.k1, m.k2ID, m.s, SecondMap.k1ID, SecondMap.k2Key, SecondMap.k3Type, SecondMap.s; SELECT d, k, m.k1, m.k2ID, sum(m.s) FROM summing_composite_key ARRAY JOIN FirstMap AS m GROUP BY d, k, m.k1, m.k2ID ORDER BY d, k, m.k1, m.k2ID; -SELECT d, k, m.k1, m.k2ID,m. s FROM summing_composite_key FINAL ARRAY JOIN FirstMap AS m ORDER BY d, k, m.k1, m.k2ID, m.s; +SELECT d, k, m.k1, m.k2ID, m.s FROM summing_composite_key FINAL ARRAY JOIN FirstMap AS m ORDER BY d, k, m.k1, m.k2ID, m.s; SELECT d, k, m.k1ID, m.k2Key, m.k3Type, m.s FROM summing_composite_key ARRAY JOIN SecondMap AS m ORDER BY d, k, m.k1ID, m.k2Key, m.k3Type, m.s; SELECT d, k, m.k1ID, m.k2Key, m.k3Type, sum(m.s) FROM summing_composite_key ARRAY JOIN SecondMap AS m GROUP BY d, k, m.k1ID, m.k2Key, m.k3Type ORDER BY d, k, m.k1ID, m.k2Key, m.k3Type; @@ -17,7 +17,7 @@ SELECT d, k, m.k1ID, m.k2Key, m.k3Type, m.s FROM summing_composite_key FINAL ARR OPTIMIZE TABLE summing_composite_key PARTITION 200001 FINAL; -SELECT * FROM summing_composite_key ORDER BY d, k, _part_index; +SELECT * FROM summing_composite_key ORDER BY d, k, FirstMap.k1, FirstMap.k2ID, FirstMap.s, SecondMap.k1ID, SecondMap.k2Key, SecondMap.k3Type, SecondMap.s;; SELECT d, k, m.k1, m.k2ID, m.s FROM summing_composite_key ARRAY JOIN FirstMap AS m ORDER BY d, k, m.k1, m.k2ID, m.s; SELECT d, k, m.k1, m.k2ID, sum(m.s) FROM summing_composite_key ARRAY JOIN FirstMap AS m GROUP BY d, k, m.k1, m.k2ID ORDER BY d, k, m.k1, m.k2ID; diff --git a/tests/queries/0_stateless/00502_custom_partitioning_local.sql b/tests/queries/0_stateless/00502_custom_partitioning_local.sql index eda267c0445..55c9b559ed0 100644 --- a/tests/queries/0_stateless/00502_custom_partitioning_local.sql +++ b/tests/queries/0_stateless/00502_custom_partitioning_local.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage SELECT '*** Not partitioned ***'; DROP TABLE IF EXISTS not_partitioned; diff --git a/tests/queries/0_stateless/00502_sum_map.sql b/tests/queries/0_stateless/00502_sum_map.sql index 51007a9c78a..3ceb5b82952 100644 --- a/tests/queries/0_stateless/00502_sum_map.sql +++ b/tests/queries/0_stateless/00502_sum_map.sql @@ -5,7 +5,7 @@ CREATE TABLE sum_map(date Date, timeslot DateTime, statusMap Nested(status UInt1 INSERT INTO sum_map VALUES ('2000-01-01', '2000-01-01 00:00:00', [1, 2, 3], [10, 10, 10]), ('2000-01-01', '2000-01-01 00:00:00', [3, 4, 5], [10, 10, 10]), ('2000-01-01', '2000-01-01 00:01:00', [4, 5, 6], [10, 10, 10]), ('2000-01-01', '2000-01-01 00:01:00', [6, 7, 8], [10, 10, 10]); -SELECT * FROM sum_map ORDER BY timeslot; +SELECT * FROM sum_map ORDER BY timeslot, statusMap.status, statusMap.requests; SELECT sumMap(statusMap.status, statusMap.requests) FROM sum_map; SELECT sumMap((statusMap.status, statusMap.requests)) FROM sum_map; SELECT sumMapMerge(s) FROM (SELECT sumMapState(statusMap.status, statusMap.requests) AS s FROM sum_map); diff --git a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh index d78f93d6bb3..cd0a4cdb640 100755 --- a/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh +++ b/tests/queries/0_stateless/00626_replace_partition_from_table_zookeeper.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: zookeeper, no-parallel +# Tags: zookeeper, no-parallel, no-s3-storage # Because REPLACE PARTITION does not forces immediate removal of replaced data parts from local filesystem # (it tries to do it as quick as possible, but it still performed in separate thread asynchronously) diff --git a/tests/queries/0_stateless/00628_in_lambda_on_merge_table_bug.sql b/tests/queries/0_stateless/00628_in_lambda_on_merge_table_bug.sql index 48a90e6fe77..ddf98149c3b 100644 --- a/tests/queries/0_stateless/00628_in_lambda_on_merge_table_bug.sql +++ b/tests/queries/0_stateless/00628_in_lambda_on_merge_table_bug.sql @@ -8,11 +8,11 @@ create table test_in_tuple as test_in_tuple_1 engine = Merge(currentDatabase(), insert into test_in_tuple_1 values (1, 1, [1, 2], [1, 2]); insert into test_in_tuple_2 values (2, 1, [1, 2], [1, 2]); -select key, arr_x, arr_y, _table from test_in_tuple left array join x as arr_x, y as arr_y order by _table; +select key, arr_x, arr_y, _table from test_in_tuple left array join x as arr_x, y as arr_y order by _table, arr_x, arr_y; select '-'; -select key, arr_x, arr_y, _table from test_in_tuple left array join x as arr_x, y as arr_y where (key_2, arr_x, arr_y) in (1, 1, 1) order by _table; +select key, arr_x, arr_y, _table from test_in_tuple left array join x as arr_x, y as arr_y where (key_2, arr_x, arr_y) in (1, 1, 1) order by _table, arr_x, arr_y; select '-'; -select key, arr_x, arr_y, _table from test_in_tuple left array join arrayFilter((t, x_0, x_1) -> (key_2, x_0, x_1) in (1, 1, 1), x, x ,y) as arr_x, arrayFilter((t, x_0, x_1) -> (key_2, x_0, x_1) in (1, 1, 1), y, x ,y) as arr_y where (key_2, arr_x, arr_y) in (1, 1, 1) order by _table; +select key, arr_x, arr_y, _table from test_in_tuple left array join arrayFilter((t, x_0, x_1) -> (key_2, x_0, x_1) in (1, 1, 1), x, x ,y) as arr_x, arrayFilter((t, x_0, x_1) -> (key_2, x_0, x_1) in (1, 1, 1), y, x ,y) as arr_y where (key_2, arr_x, arr_y) in (1, 1, 1) order by _table, arr_x, arr_y; drop table if exists test_in_tuple_1; drop table if exists test_in_tuple_2; diff --git a/tests/queries/0_stateless/00702_join_on_dups.sql b/tests/queries/0_stateless/00702_join_on_dups.sql index 852378f543f..869d39308cc 100644 --- a/tests/queries/0_stateless/00702_join_on_dups.sql +++ b/tests/queries/0_stateless/00702_join_on_dups.sql @@ -10,68 +10,68 @@ insert into Y (id, y_a) values (1, 'r1'), (1, 'r2'), (2, 'r3'), (3, 'r4'), insert into Y (id, y_a, y_b) values (4, 'r6', 'nr6'), (6, 'r7', 'nr7'), (7, 'r8', 'nr8'), (9, 'r9', 'nr9'); select 'inner'; -select X.*, Y.* from X inner join Y on X.id = Y.id order by id; +select X.*, Y.* from X inner join Y on X.id = Y.id order by X.id, X.x_a, X.x_b, Y.id, Y.y_a, Y.y_b; select 'inner subs'; -select s.*, j.* from (select * from X) as s inner join (select * from Y) as j on s.id = j.id order by id; +select s.*, j.* from (select * from X) as s inner join (select * from Y) as j on s.id = j.id order by s.id, s.x_a, s.x_b, j.id, j.y_a, j.y_b; select 'inner expr'; -select X.*, Y.* from X inner join Y on (X.id + 1) = (Y.id + 1) order by id; +select X.*, Y.* from X inner join Y on (X.id + 1) = (Y.id + 1) order by X.id, X.x_a, X.x_b, Y.id, Y.y_a, Y.y_b; select 'left'; -select X.*, Y.* from X left join Y on X.id = Y.id order by id; +select X.*, Y.* from X left join Y on X.id = Y.id order by X.id, X.x_a, X.x_b, Y.id, Y.y_a, Y.y_b; select 'left subs'; -select s.*, j.* from (select * from X) as s left join (select * from Y) as j on s.id = j.id order by id; +select s.*, j.* from (select * from X) as s left join (select * from Y) as j on s.id = j.id order by s.id, s.x_a, s.x_b, j.id, j.y_a, j.y_b; select 'left expr'; -select X.*, Y.* from X left join Y on (X.id + 1) = (Y.id + 1) order by id; +select X.*, Y.* from X left join Y on (X.id + 1) = (Y.id + 1) order by X.id, X.x_a, X.x_b, Y.id, Y.y_a, Y.y_b; select 'right'; -select X.*, Y.* from X right join Y on X.id = Y.id order by id; +select X.*, Y.* from X right join Y on X.id = Y.id order by X.id, X.x_a, X.x_b, Y.id, Y.y_a, Y.y_b; select 'right subs'; -select s.*, j.* from (select * from X) as s right join (select * from Y) as j on s.id = j.id order by id; +select s.*, j.* from (select * from X) as s right join (select * from Y) as j on s.id = j.id order by s.id, s.x_a, s.x_b, j.id, j.y_a, j.y_b; --select 'right expr'; --select X.*, Y.* from X right join Y on (X.id + 1) = (Y.id + 1) order by id; select 'full'; -select X.*, Y.* from X full join Y on X.id = Y.id order by id; +select X.*, Y.* from X full join Y on X.id = Y.id order by X.id, X.x_a, X.x_b, Y.id, Y.y_a, Y.y_b; select 'full subs'; -select s.*, j.* from (select * from X) as s full join (select * from Y) as j on s.id = j.id order by id; +select s.*, j.* from (select * from X) as s full join (select * from Y) as j on s.id = j.id order by s.id, s.x_a, s.x_b, j.id, j.y_a, j.y_b; --select 'full expr'; --select X.*, Y.* from X full join Y on (X.id + 1) = (Y.id + 1) order by id; select 'self inner'; -select X.*, s.* from X inner join (select * from X) as s on X.id = s.id order by X.id, X.x_a, s.x_a; +select X.*, s.* from X inner join (select * from X) as s on X.id = s.id order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self inner nullable'; -select X.*, s.* from X inner join (select * from X) as s on X.x_b = s.x_b order by X.id; +select X.*, s.* from X inner join (select * from X) as s on X.x_b = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self inner nullable vs not nullable'; -select X.*, s.* from X inner join (select * from X) as s on X.id = s.x_b order by X.id; +select X.*, s.* from X inner join (select * from X) as s on X.id = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; -- TODO: s.y_b == '' instead of NULL select 'self inner nullable vs not nullable 2'; -select Y.*, s.* from Y inner join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by id; +select Y.*, s.* from Y inner join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by Y.id, Y.y_a, Y.y_b, s.id, s.y_a, s.y_b; select 'self left'; -select X.*, s.* from X left join (select * from X) as s on X.id = s.id order by X.id, X.x_a, s.x_a; +select X.*, s.* from X left join (select * from X) as s on X.id = s.id order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self left nullable'; -select X.*, s.* from X left join (select * from X) as s on X.x_b = s.x_b order by X.id; +select X.*, s.* from X left join (select * from X) as s on X.x_b = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self left nullable vs not nullable'; -select X.*, s.* from X left join (select * from X) as s on X.id = s.x_b order by X.id; +select X.*, s.* from X left join (select * from X) as s on X.id = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; -- TODO: s.y_b == '' instead of NULL select 'self left nullable vs not nullable 2'; -select Y.*, s.* from Y left join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by id; +select Y.*, s.* from Y left join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by Y.id, Y.y_a, Y.y_b, s.id, s.y_a, s.y_b; select 'self right'; -select X.*, s.* from X right join (select * from X) as s on X.id = s.id order by X.id, X.x_a, s.x_a; +select X.*, s.* from X right join (select * from X) as s on X.id = s.id order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self right nullable'; -select X.*, s.* from X right join (select * from X) as s on X.x_b = s.x_b order by X.id; +select X.*, s.* from X right join (select * from X) as s on X.x_b = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self right nullable vs not nullable'; -select X.*, s.* from X right join (select * from X) as s on X.id = s.x_b order by X.id; +select X.*, s.* from X right join (select * from X) as s on X.id = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; --select 'self right nullable vs not nullable 2'; --select Y.*, s.* from Y right join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by id; select 'self full'; -select X.*, s.* from X full join (select * from X) as s on X.id = s.id order by X.id, X.x_a, s.x_a; +select X.*, s.* from X full join (select * from X) as s on X.id = s.id order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self full nullable'; -select X.*, s.* from X full join (select * from X) as s on X.x_b = s.x_b order by X.id; +select X.*, s.* from X full join (select * from X) as s on X.x_b = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; select 'self full nullable vs not nullable'; -select X.*, s.* from X full join (select * from X) as s on X.id = s.x_b order by X.id; +select X.*, s.* from X full join (select * from X) as s on X.id = s.x_b order by X.id, X.x_a, X.x_b, s.id, s.x_a, s.x_b; --select 'self full nullable vs not nullable 2'; --select Y.*, s.* from Y full join (select * from Y) as s on concat('n', Y.y_a) = s.y_b order by id; diff --git a/tests/queries/0_stateless/00702_join_with_using_dups.sql b/tests/queries/0_stateless/00702_join_with_using_dups.sql index d45c6628b9a..1499473ed11 100644 --- a/tests/queries/0_stateless/00702_join_with_using_dups.sql +++ b/tests/queries/0_stateless/00702_join_with_using_dups.sql @@ -8,24 +8,24 @@ insert into X (id, x_name) values (1, 'A'), (2, 'B'), (2, 'C'), (3, 'D'), (4, 'E insert into Y (id, y_name) values (1, 'a'), (1, 'b'), (2, 'c'), (3, 'd'), (3, 'e'), (4, 'f'), (6, 'g'), (7, 'h'), (9, 'i'); select 'inner'; -select X.*, Y.* from X inner join Y using id; +select X.*, Y.* from X inner join Y using id order by X.id, Y.id, X.x_name, Y.y_name; select 'inner subs'; -select s.*, j.* from (select * from X) as s inner join (select * from Y) as j using id; +select s.*, j.* from (select * from X) as s inner join (select * from Y) as j using id order by s.id, s.id, s.x_name, j.y_name; select 'left'; -select X.*, Y.* from X left join Y using id; +select X.*, Y.* from X left join Y using id order by X.id, Y.id, X.x_name, Y.y_name; select 'left subs'; -select s.*, j.* from (select * from X) as s left join (select * from Y) as j using id; +select s.*, j.* from (select * from X) as s left join (select * from Y) as j using id order by s.id, j.id, s.x_name, j.y_name; select 'right'; -select X.*, Y.* from X right join Y using id order by id; +select X.*, Y.* from X right join Y using id order by X.id, Y.id, X.x_name, Y.y_name; select 'right subs'; -select s.*, j.* from (select * from X) as s right join (select * from Y) as j using id order by id; +select s.*, j.* from (select * from X) as s right join (select * from Y) as j using id order by s.id, j.id, s.x_name, j.y_name; select 'full'; -select X.*, Y.* from X full join Y using id order by id; +select X.*, Y.* from X full join Y using id order by X.id, Y.id, X.x_name, Y.y_name; select 'full subs'; -select s.*, j.* from (select * from X) as s full join (select * from Y) as j using id order by id; +select s.*, j.* from (select * from X) as s full join (select * from Y) as j using id order by s.id, j.id, s.x_name, j.y_name; drop table X; drop table Y; diff --git a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh index f9e33645527..d9a3631a7dd 100755 --- a/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh +++ b/tests/queries/0_stateless/00731_long_merge_tree_select_opened_files.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: long +# Tags: long, no-s3-storage set -e @@ -30,7 +30,6 @@ $CLICKHOUSE_CLIENT $settings -q "$touching_many_parts_query" &> /dev/null $CLICKHOUSE_CLIENT $settings -q "SYSTEM FLUSH LOGS" - -$CLICKHOUSE_CLIENT $settings -q "SELECT ProfileEvents['FileOpen'] FROM system.query_log WHERE query='$touching_many_parts_query' and current_database = currentDatabase() ORDER BY event_time DESC LIMIT 1;" +$CLICKHOUSE_CLIENT $settings -q "SELECT ProfileEvents['FileOpen'] as opened_files FROM system.query_log WHERE query='$touching_many_parts_query' and current_database = currentDatabase() ORDER BY event_time DESC, opened_files DESC LIMIT 1;" $CLICKHOUSE_CLIENT $settings -q "DROP TABLE IF EXISTS merge_tree_table;" diff --git a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql index dc54e4630d6..ca3df7c4cd4 100644 --- a/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql +++ b/tests/queries/0_stateless/00753_system_columns_and_system_tables_long.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-s3-storage DROP TABLE IF EXISTS check_system_tables; diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by.reference b/tests/queries/0_stateless/00754_alter_modify_order_by.reference index 0279e5ca11b..bcbf66e824d 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by.reference +++ b/tests/queries/0_stateless/00754_alter_modify_order_by.reference @@ -1,9 +1,7 @@ *** Check that the parts are sorted according to the new key. *** -1 2 0 10 -1 2 0 20 1 2 2 40 -1 2 2 50 1 2 1 30 +1 2 0 10 *** Check that the rows are collapsed according to the new key. *** 1 2 0 30 1 2 1 30 diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by.sql b/tests/queries/0_stateless/00754_alter_modify_order_by.sql index a09d824c928..cb81f868e7b 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by.sql +++ b/tests/queries/0_stateless/00754_alter_modify_order_by.sql @@ -21,14 +21,14 @@ ALTER TABLE summing MODIFY ORDER BY (x, y, nonexistent); -- { serverError 47} /* Can't modyfy ORDER BY so that it is no longer a prefix of the PRIMARY KEY. */ ALTER TABLE summing MODIFY ORDER BY x; -- { serverError 36} -INSERT INTO summing(x, y, val) VALUES (1, 2, 10), (1, 2, 20); - ALTER TABLE summing ADD COLUMN z UInt32 AFTER y, MODIFY ORDER BY (x, y, -z); -INSERT INTO summing(x, y, z, val) values (1, 2, 1, 30), (1, 2, 2, 40), (1, 2, 2, 50); +INSERT INTO summing(x, y, z, val) values (1, 2, 0, 10), (1, 2, 1, 30), (1, 2, 2, 40); SELECT '*** Check that the parts are sorted according to the new key. ***'; -SELECT * FROM summing ORDER BY _part; +SELECT * FROM summing; + +INSERT INTO summing(x, y, z, val) values (1, 2, 0, 20), (1, 2, 2, 50); SELECT '*** Check that the rows are collapsed according to the new key. ***'; SELECT * FROM summing FINAL ORDER BY x, y, z; diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.reference b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.reference index a1fecd72e30..6d8a3f181e4 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.reference +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.reference @@ -1,9 +1,7 @@ *** Check that the parts are sorted according to the new key. *** -1 2 0 10 -1 2 0 20 1 2 2 40 -1 2 2 50 1 2 1 30 +1 2 0 10 *** Check that the rows are collapsed according to the new key. *** 1 2 0 30 1 2 1 30 diff --git a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.sql b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.sql index 8c19e30249f..c859c7b9921 100644 --- a/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.sql +++ b/tests/queries/0_stateless/00754_alter_modify_order_by_replicated_zookeeper_long.sql @@ -27,16 +27,16 @@ ALTER TABLE summing_r1 MODIFY ORDER BY (x, y, nonexistent); -- { serverError 47 /* Can't modyfy ORDER BY so that it is no longer a prefix of the PRIMARY KEY. */ ALTER TABLE summing_r1 MODIFY ORDER BY x; -- { serverError 36 } -INSERT INTO summing_r1(x, y, val) VALUES (1, 2, 10), (1, 2, 20); -SYSTEM SYNC REPLICA summing_r2; - ALTER TABLE summing_r1 ADD COLUMN z UInt32 AFTER y, MODIFY ORDER BY (x, y, -z); -INSERT INTO summing_r1(x, y, z, val) values (1, 2, 1, 30), (1, 2, 2, 40), (1, 2, 2, 50); +INSERT INTO summing_r1(x, y, z, val) values (1, 2, 0, 10), (1, 2, 1, 30), (1, 2, 2, 40); SYSTEM SYNC REPLICA summing_r2; SELECT '*** Check that the parts are sorted according to the new key. ***'; -SELECT * FROM summing_r2 ORDER BY _part; +SELECT * FROM summing_r2; + +INSERT INTO summing_r1(x, y, z, val) values (1, 2, 0, 20), (1, 2, 2, 50); +SYSTEM SYNC REPLICA summing_r2; SELECT '*** Check that the rows are collapsed according to the new key. ***'; SELECT * FROM summing_r2 FINAL ORDER BY x, y, z; diff --git a/tests/queries/0_stateless/00799_function_dry_run.reference b/tests/queries/0_stateless/00799_function_dry_run.reference index 35cebe7569a..8f855bb582d 100644 --- a/tests/queries/0_stateless/00799_function_dry_run.reference +++ b/tests/queries/0_stateless/00799_function_dry_run.reference @@ -2,8 +2,8 @@ 0.3 2018-11-19 13:05:00 \N 0.4 2018-11-19 13:10:00 1 0.5 2018-11-19 13:15:00 1.2 -0.6 2018-11-19 13:15:00 1.5 -0.7 2018-11-19 13:20:00 1.8 -0.8 2018-11-19 13:25:00 2.1 -0.9 2018-11-19 13:25:00 2.4 -0.5 2018-11-19 13:30:00 2.2 +0.6 2018-11-19 13:20:00 1.5 +0.7 2018-11-19 13:25:00 1.8 +0.8 2018-11-19 13:30:00 2.1 +0.9 2018-11-19 13:45:00 2.4 +0.5 2018-11-19 13:50:00 2.2 diff --git a/tests/queries/0_stateless/00799_function_dry_run.sql b/tests/queries/0_stateless/00799_function_dry_run.sql index 4f3df6a0ff8..bf6fb3de395 100644 --- a/tests/queries/0_stateless/00799_function_dry_run.sql +++ b/tests/queries/0_stateless/00799_function_dry_run.sql @@ -4,7 +4,7 @@ DROP TABLE IF EXISTS bm; CREATE TABLE bm (amount float, business_dttm DateTime) engine Log; -INSERT INTO bm VALUES (0.3,'2018-11-19 13:00:00'), (0.3,'2018-11-19 13:05:00'), (0.4,'2018-11-19 13:10:00'), (0.5,'2018-11-19 13:15:00'), (0.6,'2018-11-19 13:15:00'), (0.7,'2018-11-19 13:20:00'), (0.8,'2018-11-19 13:25:00'), (0.9,'2018-11-19 13:25:00'), (0.5,'2018-11-19 13:30:00'); +INSERT INTO bm VALUES (0.3,'2018-11-19 13:00:00'), (0.3,'2018-11-19 13:05:00'), (0.4,'2018-11-19 13:10:00'), (0.5,'2018-11-19 13:15:00'), (0.6,'2018-11-19 13:20:00'), (0.7,'2018-11-19 13:25:00'), (0.8,'2018-11-19 13:30:00'), (0.9,'2018-11-19 13:45:00'), (0.5,'2018-11-19 13:50:00'); WITH ( @@ -30,6 +30,6 @@ FROM business_dttm FROM bm ORDER BY business_dttm -); +) ORDER BY business_dttm; DROP TABLE bm; diff --git a/tests/queries/0_stateless/00818_inner_join_bug_3567.sql b/tests/queries/0_stateless/00818_inner_join_bug_3567.sql index 2058d2309e4..1c851d40f47 100644 --- a/tests/queries/0_stateless/00818_inner_join_bug_3567.sql +++ b/tests/queries/0_stateless/00818_inner_join_bug_3567.sql @@ -9,8 +9,8 @@ INSERT INTO table2 VALUES ('D', 'd', '2018-01-01') ('B', 'b', '2018-01-01') ('C' SELECT * FROM table1 t1 FORMAT PrettyCompact; SELECT *, c as a, d as b FROM table2 FORMAT PrettyCompact; -SELECT * FROM table1 t1 ALL LEFT JOIN (SELECT *, c, d as b FROM table2) t2 USING (a, b) ORDER BY d FORMAT PrettyCompact; -SELECT * FROM table1 t1 ALL INNER JOIN (SELECT *, c, d as b FROM table2) t2 USING (a, b) ORDER BY d FORMAT PrettyCompact; +SELECT * FROM table1 t1 ALL LEFT JOIN (SELECT *, c, d as b FROM table2) t2 USING (a, b) ORDER BY d, t1.a FORMAT PrettyCompact; +SELECT * FROM table1 t1 ALL INNER JOIN (SELECT *, c, d as b FROM table2) t2 USING (a, b) ORDER BY d, t1.a FORMAT PrettyCompact; DROP TABLE table1; DROP TABLE table2; diff --git a/tests/queries/0_stateless/00853_join_with_nulls_crash.sql b/tests/queries/0_stateless/00853_join_with_nulls_crash.sql index 464ddbb1990..c63c2d99cba 100644 --- a/tests/queries/0_stateless/00853_join_with_nulls_crash.sql +++ b/tests/queries/0_stateless/00853_join_with_nulls_crash.sql @@ -21,37 +21,37 @@ SELECT s1.other, s2.other, count_a, count_b, toTypeName(s1.other), toTypeName(s2 ALL FULL JOIN ( SELECT other, count() AS count_b FROM table_b GROUP BY other ) s2 ON s1.other = s2.other -ORDER BY s2.other DESC, count_a; +ORDER BY s2.other DESC, count_a, s1.other; SELECT s1.other, s2.other, count_a, count_b, toTypeName(s1.other), toTypeName(s2.other) FROM ( SELECT other, count() AS count_a FROM table_a GROUP BY other ) s1 ALL FULL JOIN ( SELECT other, count() AS count_b FROM table_b GROUP BY other ) s2 USING other -ORDER BY s2.other DESC, count_a; +ORDER BY s2.other DESC, count_a, s1.other; SELECT s1.something, s2.something, count_a, count_b, toTypeName(s1.something), toTypeName(s2.something) FROM ( SELECT something, count() AS count_a FROM table_a GROUP BY something ) s1 ALL FULL JOIN ( SELECT something, count() AS count_b FROM table_b GROUP BY something ) s2 ON s1.something = s2.something -ORDER BY count_a DESC; +ORDER BY count_a DESC, something, s2.something; SELECT s1.something, s2.something, count_a, count_b, toTypeName(s1.something), toTypeName(s2.something) FROM ( SELECT something, count() AS count_a FROM table_a GROUP BY something ) s1 ALL RIGHT JOIN ( SELECT something, count() AS count_b FROM table_b GROUP BY something ) s2 USING (something) -ORDER BY count_a DESC; +ORDER BY count_a DESC, s1.something, s2.something; SET joined_subquery_requires_alias = 0; SELECT something, count_a, count_b, toTypeName(something) FROM - ( SELECT something, count() AS count_a FROM table_a GROUP BY something ) + ( SELECT something, count() AS count_a FROM table_a GROUP BY something ) as s1 ALL FULL JOIN - ( SELECT something, count() AS count_b FROM table_b GROUP BY something ) + ( SELECT something, count() AS count_b FROM table_b GROUP BY something ) as s2 USING (something) -ORDER BY count_a DESC; +ORDER BY count_a DESC, something DESC; DROP TABLE table_a; DROP TABLE table_b; diff --git a/tests/queries/0_stateless/00909_ngram_distance.reference b/tests/queries/0_stateless/00909_ngram_distance.reference index 52fb462a9ed..290e24faac5 100644 --- a/tests/queries/0_stateless/00909_ngram_distance.reference +++ b/tests/queries/0_stateless/00909_ngram_distance.reference @@ -110,116 +110,116 @@ 77 636 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metrica.yandex.com/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrica.yandex.com/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 0 привет как дела?... Херсон 600 пап привет как дела - Яндекс.Видео 684 привет как дела клип - Яндекс.Видео 692 привет братан как дела - Яндекс.Видео 707 -привет 1000 -http://metric.ru/ 1000 http://autometric.ru/ 1000 +http://metric.ru/ 1000 http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 http://metrika.ru/ 1000 +http://metris.ru/ 1000 +привет 1000 0 http://metric.ru/ 765 http://metris.ru/ 765 http://metrika.ru/ 778 http://autometric.ru/ 810 http://metrica.yandex.com/ 846 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 пап привет как дела - Яндекс.Видео 1000 +привет 1000 привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 привет как дела?... Херсон 297 пап привет как дела - Яндекс.Видео 422 привет как дела клип - Яндекс.Видео 435 привет братан как дела - Яндекс.Видео 500 привет 529 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 привет как дела?... Херсон 459 пап привет как дела - Яндекс.Видео 511 привет 529 привет как дела клип - Яндекс.Видео 565 привет братан как дела - Яндекс.Видео 583 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 http://metrika.ru/ 524 http://metric.ru/ 700 http://metris.ru/ 700 http://autometric.ru/ 750 http://metrica.yandex.com/ 793 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metric.ru/ 600 http://metrica.yandex.com/ 655 http://autometric.ru/ 667 http://metris.ru/ 700 http://metrika.ru/ 714 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrika.ru/ 619 http://metric.ru/ 700 http://metris.ru/ 700 http://autometric.ru/ 750 http://metrica.yandex.com/ 793 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metric.ru/ 600 http://autometric.ru/ 667 http://metris.ru/ 700 http://metrika.ru/ 714 http://metrica.yandex.com/ 724 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrica.yandex.com/ 714 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 0 0 0 @@ -332,138 +332,138 @@ http://metrika.ru/ 1000 77 636 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metrica.yandex.com/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrica.yandex.com/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 0 привет как дела?... Херсон 600 пап привет как дела - Яндекс.Видео 684 привет как дела клип - Яндекс.Видео 692 привет братан как дела - Яндекс.Видео 707 -привет 1000 -http://metric.ru/ 1000 http://autometric.ru/ 1000 +http://metric.ru/ 1000 http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 http://metrika.ru/ 1000 +http://metris.ru/ 1000 +привет 1000 0 http://metric.ru/ 765 http://metris.ru/ 765 http://metrika.ru/ 778 http://autometric.ru/ 810 http://metrica.yandex.com/ 846 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 пап привет как дела - Яндекс.Видео 1000 +привет 1000 привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 привет как дела?... Херсон 297 пап привет как дела - Яндекс.Видео 422 привет как дела клип - Яндекс.Видео 435 привет братан как дела - Яндекс.Видео 500 привет 529 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 привет как дела?... Херсон 459 пап привет как дела - Яндекс.Видео 511 привет 529 привет как дела клип - Яндекс.Видео 565 привет братан как дела - Яндекс.Видео 583 -http://metric.ru/ 1000 + 1000 http://autometric.ru/ 1000 +http://metric.ru/ 1000 http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 http://metrika.ru/ 1000 - 1000 +http://metris.ru/ 1000 http://metrika.ru/ 524 http://metric.ru/ 700 http://metris.ru/ 700 http://autometric.ru/ 750 http://metrica.yandex.com/ 793 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrika.ru/ 524 http://metric.ru/ 700 http://metris.ru/ 700 http://autometric.ru/ 750 http://metrica.yandex.com/ 793 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metric.ru/ 600 http://metrica.yandex.com/ 655 http://autometric.ru/ 667 http://metris.ru/ 700 http://metrika.ru/ 714 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrika.ru/ 619 http://metric.ru/ 700 http://metris.ru/ 700 http://autometric.ru/ 750 http://metrica.yandex.com/ 793 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metric.ru/ 600 http://autometric.ru/ 667 http://metris.ru/ 700 http://metrika.ru/ 714 http://metrica.yandex.com/ 724 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrica.yandex.com/ 714 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 привет как дела клип - Яндекс.Видео 0 пап привет как дела - Яндекс.Видео 169 привет братан как дела - Яндекс.Видео 235 привет как дела?... Херсон 544 привет 784 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 0 0 0 @@ -581,78 +581,78 @@ http://metrika.ru/ 1000 привет как дела клип - Яндекс.Видео 412 привет братан как дела - Яндекс.Видео 461 привет 471 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 привет как дела?... Херсон 343 пап привет как дела - Яндекс.Видео 446 привет 471 привет как дела клип - Яндекс.Видео 482 привет братан как дела - Яндекс.Видео 506 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 http://metrika.ru/ 579 http://metric.ru/ 778 http://metris.ru/ 778 http://autometric.ru/ 818 http://metrica.yandex.com/ 852 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metric.ru/ 667 http://metrica.yandex.com/ 704 http://autometric.ru/ 727 http://metris.ru/ 778 http://metrika.ru/ 789 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrika.ru/ 684 http://metric.ru/ 778 http://metris.ru/ 778 http://autometric.ru/ 818 http://metrica.yandex.com/ 852 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metric.ru/ 667 http://autometric.ru/ 727 http://metrica.yandex.com/ 778 http://metris.ru/ 778 http://metrika.ru/ 789 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrica.yandex.com/ 769 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 0 0 0 @@ -769,87 +769,87 @@ http://metrika.ru/ 1000 пап привет как дела - Яндекс.Видео 928 привет как дела клип - Яндекс.Видео 929 привет братан как дела - Яндекс.Видео 955 -привет 1000 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +привет 1000 привет как дела?... Херсон 672 пап привет как дела - Яндекс.Видео 735 привет как дела клип - Яндекс.Видео 741 привет братан как дела - Яндекс.Видео 753 -привет 1000 -http://metric.ru/ 1000 + 1000 http://autometric.ru/ 1000 +http://metric.ru/ 1000 http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 http://metrika.ru/ 1000 - 1000 +http://metris.ru/ 1000 +привет 1000 http://metrika.ru/ 579 http://metric.ru/ 778 http://metris.ru/ 778 http://autometric.ru/ 818 http://metrica.yandex.com/ 852 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrika.ru/ 579 http://metric.ru/ 778 http://metris.ru/ 778 http://autometric.ru/ 818 http://metrica.yandex.com/ 852 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metric.ru/ 667 http://metrica.yandex.com/ 704 http://autometric.ru/ 727 http://metris.ru/ 778 http://metrika.ru/ 789 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrika.ru/ 684 http://metric.ru/ 778 http://metris.ru/ 778 http://autometric.ru/ 818 http://metrica.yandex.com/ 852 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metric.ru/ 667 http://autometric.ru/ 727 http://metrica.yandex.com/ 778 http://metris.ru/ 778 http://metrika.ru/ 789 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 http://metrica.yandex.com/ 769 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 diff --git a/tests/queries/0_stateless/00909_ngram_distance.sql b/tests/queries/0_stateless/00909_ngram_distance.sql index ed800bf6c97..b2f403c415a 100644 --- a/tests/queries/0_stateless/00909_ngram_distance.sql +++ b/tests/queries/0_stateless/00909_ngram_distance.sql @@ -34,17 +34,17 @@ drop table if exists test_distance; create table test_distance (Title String) engine = Memory; insert into test_distance values ('привет как дела?... Херсон'), ('привет как дела клип - Яндекс.Видео'), ('привет'), ('пап привет как дела - Яндекс.Видео'), ('привет братан как дела - Яндекс.Видео'), ('http://metric.ru/'), ('http://autometric.ru/'), ('http://metrica.yandex.com/'), ('http://metris.ru/'), ('http://metrika.ru/'), (''); -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, Title) as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, extract(Title, 'как дела')) as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, extract(Title, 'metr')) as distance; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, Title) as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, extract(Title, 'как дела')) as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, extract(Title, 'metr')) as distance, Title; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'привет как дела') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'как привет дела') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'metrica') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'metriks') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'metrics') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'yandex') as distance; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'привет как дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'как привет дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'metrica') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'metriks') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'metrics') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceUTF8(Title, 'yandex') as distance, Title; select round(1000 * ngramDistanceCaseInsensitiveUTF8(materialize(''), '')) from system.numbers limit 5; @@ -80,19 +80,19 @@ select round(1000 * ngramDistanceCaseInsensitiveUTF8('аБВГдеёЖз', 'Аб select round(1000 * ngramDistanceCaseInsensitiveUTF8('абвгдеёжз', 'гдеёЗД')); select round(1000 * ngramDistanceCaseInsensitiveUTF8('АБВГДеёжз', 'ЁЁЁЁЁЁЁЁ')); -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, Title) as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, extract(Title, 'как дела')) as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, extract(Title, 'metr')) as distance; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, Title) as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, extract(Title, 'как дела')) as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, extract(Title, 'metr')) as distance, Title; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'ПрИвЕт кАК ДЕЛа') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'как ПРИВЕТ дела') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'Metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'mEtrica') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metriKS') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metrics') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'YanDEX') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'приВЕТ КАк ДеЛа КлИп - яндеКс.видео') as distance; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'ПрИвЕт кАК ДЕЛа') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'как ПРИВЕТ дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'Metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'mEtrica') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metriKS') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'metrics') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'YanDEX') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitiveUTF8(Title, 'приВЕТ КАк ДеЛа КлИп - яндеКс.видео') as distance, Title; select round(1000 * ngramDistance(materialize(''), '')) from system.numbers limit 5; @@ -128,13 +128,13 @@ select round(1000 * ngramDistance('abcdefgh', 'abcdefg')); select round(1000 * ngramDistance('abcdefgh', 'defgh')); select round(1000 * ngramDistance('abcdefgh', 'aaaaaaaa')); -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'привет как дела') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'как привет дела') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'metrica') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'metriks') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'metrics') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'yandex') as distance; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'привет как дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'как привет дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'metrica') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'metriks') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'metrics') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistance(Title, 'yandex') as distance, Title; select round(1000 * ngramDistanceCaseInsensitive(materialize(''), '')) from system.numbers limit 5; select round(1000 * ngramDistanceCaseInsensitive(materialize('abc'), '')) from system.numbers limit 5; @@ -168,13 +168,13 @@ select round(1000 * ngramDistanceCaseInsensitive('abcdefgh', 'abcdeFG')); select round(1000 * ngramDistanceCaseInsensitive('AAAAbcdefgh', 'defgh')); select round(1000 * ngramDistanceCaseInsensitive('ABCdefgH', 'aaaaaaaa')); -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'ПрИвЕт кАК ДЕЛа') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'как ПРИВЕТ дела') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'Metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'mEtrica') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metriKS') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metrics') as distance; -SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'YanDEX') as distance; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'ПрИвЕт кАК ДЕЛа') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'как ПРИВЕТ дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'Metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'mEtrica') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metriKS') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'metrics') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_distance ORDER BY ngramDistanceCaseInsensitive(Title, 'YanDEX') as distance, Title; drop table if exists test_distance; diff --git a/tests/queries/0_stateless/00926_geo_to_h3.reference b/tests/queries/0_stateless/00926_geo_to_h3.reference index ad594f0e81f..074584ead16 100644 --- a/tests/queries/0_stateless/00926_geo_to_h3.reference +++ b/tests/queries/0_stateless/00926_geo_to_h3.reference @@ -4,12 +4,12 @@ 644325528491955313 644325528627451570 644325529094369568 -644325528491955313 +639821928864584823 644325528491955313 644325528491955313 644325528627451570 644325529094369568 -55.720762 37.598135 644325528491955313 +55.720762 37.598135 639821928864584823 55.720762 37.598135 644325528491955313 55.72076201 37.598135 644325528491955313 55.763241 37.660183 644325528627451570 diff --git a/tests/queries/0_stateless/00926_geo_to_h3.sql b/tests/queries/0_stateless/00926_geo_to_h3.sql index ed8e154fd9e..a86548d3555 100644 --- a/tests/queries/0_stateless/00926_geo_to_h3.sql +++ b/tests/queries/0_stateless/00926_geo_to_h3.sql @@ -11,9 +11,10 @@ INSERT INTO table1 VALUES(55.72076201, 37.59813500, 15); INSERT INTO table1 VALUES(55.72076200, 37.59813500, 14); select geoToH3(37.63098076, 55.77922738, 15); +select geoToH3(37.63098076, 55.77922738, 24); -- { serverError 69 } select geoToH3(lon, lat, resolution) from table1 order by lat, lon, resolution; -select geoToH3(lon, lat, 15) AS k from table1 order by lat, lon, k; -select lat, lon, geoToH3(lon, lat, 15) AS k from table1 order by lat, lon, k; +select geoToH3(lon, lat, resolution) AS k from table1 order by lat, lon, k; +select lat, lon, geoToH3(lon, lat, resolution) AS k from table1 order by lat, lon, k; select geoToH3(lon, lat, resolution) AS k, count(*) from table1 group by k order by k; DROP TABLE table1 diff --git a/tests/queries/0_stateless/00933_alter_ttl.sql b/tests/queries/0_stateless/00933_alter_ttl.sql index 4f586bb20fa..934d33660de 100644 --- a/tests/queries/0_stateless/00933_alter_ttl.sql +++ b/tests/queries/0_stateless/00933_alter_ttl.sql @@ -13,7 +13,7 @@ insert into ttl values (toDateTime('2100-10-10 00:00:00'), 3); insert into ttl values (toDateTime('2100-10-10 00:00:00'), 4); optimize table ttl partition 10 final; -select * from ttl order by d; +select * from ttl order by d, a; alter table ttl modify ttl a; -- { serverError 450 } diff --git a/tests/queries/0_stateless/00951_ngram_search.reference b/tests/queries/0_stateless/00951_ngram_search.reference index 1b845b6015d..a98f63a198a 100644 --- a/tests/queries/0_stateless/00951_ngram_search.reference +++ b/tests/queries/0_stateless/00951_ngram_search.reference @@ -110,115 +110,115 @@ 1000 500 0 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 -http://metric.ru/ 1000 http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 - 1000 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 http://metric.ru/ 1000 -http://autometric.ru/ 1000 http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 1000 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metrica.yandex.com/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 + 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrica.yandex.com/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 привет 308 привет братан как дела - Яндекс.Видео 923 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 пап привет как дела - Яндекс.Видео 1000 -http://metric.ru/ 0 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 + 0 http://autometric.ru/ 0 +http://metric.ru/ 0 http://metrica.yandex.com/ 0 -http://metris.ru/ 0 http://metrika.ru/ 0 - 0 +http://metris.ru/ 0 привет 308 -привет как дела?... Херсон 769 -привет как дела клип - Яндекс.Видео 769 привет братан как дела - Яндекс.Видео 769 +привет как дела клип - Яндекс.Видео 769 +привет как дела?... Херсон 769 пап привет как дела - Яндекс.Видео 846 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 600 +http://metric.ru/ 600 http://metrica.yandex.com/ 600 http://metris.ru/ 600 http://metrika.ru/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metris.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrika.ru/ 600 -http://metric.ru/ 800 +http://metris.ru/ 600 http://autometric.ru/ 800 +http://metric.ru/ 800 http://metrica.yandex.com/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 600 +http://metric.ru/ 600 http://metrica.yandex.com/ 600 http://metris.ru/ 600 http://metrika.ru/ 800 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metris.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrika.ru/ 600 -http://metric.ru/ 800 +http://metris.ru/ 600 http://autometric.ru/ 800 +http://metric.ru/ 800 http://metrica.yandex.com/ 800 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrica.yandex.com/ 1000 1000 1000 @@ -332,133 +332,133 @@ http://metrica.yandex.com/ 1000 1000 500 0 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 -http://metric.ru/ 1000 -http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 1000 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 -http://metric.ru/ 1000 http://autometric.ru/ 1000 -http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 -http://metrika.ru/ 1000 - 1000 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 -привет 1000 -пап привет как дела - Яндекс.Видео 1000 -привет братан как дела - Яндекс.Видео 1000 http://metric.ru/ 1000 -http://autometric.ru/ 1000 http://metrica.yandex.com/ 1000 -http://metris.ru/ 1000 http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 1000 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metrica.yandex.com/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 + 1000 +http://autometric.ru/ 1000 +http://metric.ru/ 1000 +http://metrica.yandex.com/ 1000 +http://metrika.ru/ 1000 +http://metris.ru/ 1000 +пап привет как дела - Яндекс.Видео 1000 +привет 1000 +привет братан как дела - Яндекс.Видео 1000 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrica.yandex.com/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 привет 308 привет братан как дела - Яндекс.Видео 923 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 пап привет как дела - Яндекс.Видео 1000 -http://metric.ru/ 0 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 + 0 http://autometric.ru/ 0 +http://metric.ru/ 0 http://metrica.yandex.com/ 0 -http://metris.ru/ 0 http://metrika.ru/ 0 - 0 +http://metris.ru/ 0 привет 308 -привет как дела?... Херсон 769 -привет как дела клип - Яндекс.Видео 769 привет братан как дела - Яндекс.Видео 769 +привет как дела клип - Яндекс.Видео 769 +привет как дела?... Херсон 769 пап привет как дела - Яндекс.Видео 846 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 600 +http://metric.ru/ 600 http://metrica.yandex.com/ 600 http://metris.ru/ 600 http://metrika.ru/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 600 +http://metric.ru/ 600 http://metrica.yandex.com/ 600 http://metris.ru/ 600 http://metrika.ru/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metris.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrika.ru/ 600 -http://metric.ru/ 800 +http://metris.ru/ 600 http://autometric.ru/ 800 +http://metric.ru/ 800 http://metrica.yandex.com/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 600 +http://metric.ru/ 600 http://metrica.yandex.com/ 600 http://metris.ru/ 600 http://metrika.ru/ 800 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metris.ru/ 600 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrika.ru/ 600 -http://metric.ru/ 800 +http://metris.ru/ 600 http://autometric.ru/ 800 +http://metric.ru/ 800 http://metrica.yandex.com/ 800 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 + 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 пап привет как дела - Яндекс.Видео 0 +привет 0 привет братан как дела - Яндекс.Видео 0 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 - 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrica.yandex.com/ 1000 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metrica.yandex.com/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrica.yandex.com/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 привет 121 привет как дела?... Херсон 394 привет братан как дела - Яндекс.Видео 788 @@ -576,82 +576,82 @@ http://metrika.ru/ 0 1000 1000 1000 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metrica.yandex.com/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrica.yandex.com/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 привет 360 привет братан как дела - Яндекс.Видео 960 -привет как дела?... Херсон 1000 -привет как дела клип - Яндекс.Видео 1000 пап привет как дела - Яндекс.Видео 1000 -http://metric.ru/ 0 +привет как дела клип - Яндекс.Видео 1000 +привет как дела?... Херсон 1000 + 0 http://autometric.ru/ 0 +http://metric.ru/ 0 http://metrica.yandex.com/ 0 -http://metris.ru/ 0 http://metrika.ru/ 0 - 0 +http://metris.ru/ 0 привет 360 -привет как дела?... Херсон 880 -привет как дела клип - Яндекс.Видео 880 привет братан как дела - Яндекс.Видео 880 +привет как дела клип - Яндекс.Видео 880 +привет как дела?... Херсон 880 пап привет как дела - Яндекс.Видео 920 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 500 +http://metric.ru/ 500 http://metrica.yandex.com/ 500 http://metris.ru/ 500 http://metrika.ru/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metris.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrika.ru/ 500 -http://metric.ru/ 750 +http://metris.ru/ 500 http://autometric.ru/ 750 +http://metric.ru/ 750 http://metrica.yandex.com/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 500 +http://metric.ru/ 500 http://metrica.yandex.com/ 500 http://metris.ru/ 500 http://metrika.ru/ 750 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metris.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrika.ru/ 500 -http://metric.ru/ 750 +http://metris.ru/ 500 http://autometric.ru/ 750 +http://metric.ru/ 750 http://metrica.yandex.com/ 750 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrica.yandex.com/ 1000 1000 1000 @@ -765,91 +765,91 @@ http://metrica.yandex.com/ 1000 1000 1000 1000 -привет 0 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metrica.yandex.com/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrica.yandex.com/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 +привет 0 привет братан как дела - Яндекс.Видео 80 -привет как дела?... Херсон 120 -привет как дела клип - Яндекс.Видео 120 пап привет как дела - Яндекс.Видео 120 -привет 0 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metrica.yandex.com/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 +привет как дела клип - Яндекс.Видео 120 +привет как дела?... Херсон 120 0 -привет как дела?... Херсон 440 -привет как дела клип - Яндекс.Видео 440 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrica.yandex.com/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 +привет 0 пап привет как дела - Яндекс.Видео 440 привет братан как дела - Яндекс.Видео 440 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 440 +привет как дела?... Херсон 440 0 -http://metric.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 500 +http://metric.ru/ 500 http://metrica.yandex.com/ 500 http://metris.ru/ 500 http://metrika.ru/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 500 +http://metric.ru/ 500 http://metrica.yandex.com/ 500 http://metris.ru/ 500 http://metrika.ru/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metris.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrika.ru/ 500 -http://metric.ru/ 750 +http://metris.ru/ 500 http://autometric.ru/ 750 +http://metric.ru/ 750 http://metrica.yandex.com/ 1000 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metric.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://autometric.ru/ 500 +http://metric.ru/ 500 http://metrica.yandex.com/ 500 http://metris.ru/ 500 http://metrika.ru/ 750 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 0 -http://metris.ru/ 500 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrika.ru/ 500 -http://metric.ru/ 750 +http://metris.ru/ 500 http://autometric.ru/ 750 +http://metric.ru/ 750 http://metrica.yandex.com/ 750 -привет как дела?... Херсон 0 -привет как дела клип - Яндекс.Видео 0 -привет 0 -пап привет как дела - Яндекс.Видео 0 -привет братан как дела - Яндекс.Видео 0 -http://metric.ru/ 0 -http://autometric.ru/ 0 -http://metris.ru/ 0 -http://metrika.ru/ 0 0 +http://autometric.ru/ 0 +http://metric.ru/ 0 +http://metrika.ru/ 0 +http://metris.ru/ 0 +пап привет как дела - Яндекс.Видео 0 +привет 0 +привет братан как дела - Яндекс.Видео 0 +привет как дела клип - Яндекс.Видео 0 +привет как дела?... Херсон 0 http://metrica.yandex.com/ 1000 diff --git a/tests/queries/0_stateless/00951_ngram_search.sql b/tests/queries/0_stateless/00951_ngram_search.sql index ea98e89964c..f1a37605ebc 100644 --- a/tests/queries/0_stateless/00951_ngram_search.sql +++ b/tests/queries/0_stateless/00951_ngram_search.sql @@ -34,17 +34,17 @@ drop table if exists test_entry_distance; create table test_entry_distance (Title String) engine = Memory; insert into test_entry_distance values ('привет как дела?... Херсон'), ('привет как дела клип - Яндекс.Видео'), ('привет'), ('пап привет как дела - Яндекс.Видео'), ('привет братан как дела - Яндекс.Видео'), ('http://metric.ru/'), ('http://autometric.ru/'), ('http://metrica.yandex.com/'), ('http://metris.ru/'), ('http://metrika.ru/'), (''); -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, Title) as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, extract(Title, 'как дела')) as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, extract(Title, 'metr')) as distance; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, Title) as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, extract(Title, 'как дела')) as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, extract(Title, 'metr')) as distance, Title; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'привет как дела') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'как привет дела') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'metrica') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'metriks') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'metrics') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'yandex') as distance; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'привет как дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'как привет дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'metrica') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'metriks') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'metrics') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchUTF8(Title, 'yandex') as distance, Title; select round(1000 * ngramSearchCaseInsensitiveUTF8(materialize(''), '')) from system.numbers limit 5; @@ -80,19 +80,19 @@ select round(1000 * ngramSearchCaseInsensitiveUTF8('аБВГдеёЖз', 'Абв select round(1000 * ngramSearchCaseInsensitiveUTF8('абвгдеёжз', 'гдеёЗД')); select round(1000 * ngramSearchCaseInsensitiveUTF8('АБВГДеёжз', 'ЁЁЁЁЁЁЁЁ')); -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, Title) as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, extract(Title, 'как дела')) as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, extract(Title, 'metr')) as distance; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, Title) as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, extract(Title, 'как дела')) as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, extract(Title, 'metr')) as distance, Title; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'ПрИвЕт кАК ДЕЛа') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'как ПРИВЕТ дела') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'Metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'mEtrica') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'metriKS') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'metrics') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'YanDEX') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'приВЕТ КАк ДеЛа КлИп - яндеКс.видео') as distance; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'ПрИвЕт кАК ДЕЛа') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'как ПРИВЕТ дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'Metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'mEtrica') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'metriKS') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'metrics') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'YanDEX') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitiveUTF8(Title, 'приВЕТ КАк ДеЛа КлИп - яндеКс.видео') as distance, Title; select round(1000 * ngramSearch(materialize(''), '')) from system.numbers limit 5; @@ -128,13 +128,13 @@ select round(1000 * ngramSearch('abcdefgh', 'abcdefg')); select round(1000 * ngramSearch('abcdefgh', 'defgh')); select round(1000 * ngramSearch('abcdefghaaaaaaaaaa', 'aaaaaaaa')); -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'привет как дела') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'как привет дела') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'metrica') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'metriks') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'metrics') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'yandex') as distance; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'привет как дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'как привет дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'metrica') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'metriks') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'metrics') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearch(Title, 'yandex') as distance, Title; select round(1000 * ngramSearchCaseInsensitive(materialize(''), '')) from system.numbers limit 5; select round(1000 * ngramSearchCaseInsensitive(materialize('abc'), '')) from system.numbers limit 5; @@ -168,13 +168,13 @@ select round(1000 * ngramSearchCaseInsensitive('abcdefgh', 'abcdeFG')); select round(1000 * ngramSearchCaseInsensitive('AAAAbcdefgh', 'defgh')); select round(1000 * ngramSearchCaseInsensitive('ABCdefgHaAaaaAaaaAA', 'aaaaaaaa')); -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'ПрИвЕт кАК ДЕЛа') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'как ПРИВЕТ дела') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'Metrika') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'mEtrica') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'metriKS') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'metrics') as distance; -SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'YanDEX') as distance; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'ПрИвЕт кАК ДЕЛа') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'как ПРИВЕТ дела') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'Metrika') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'mEtrica') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'metriKS') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'metrics') as distance, Title; +SELECT Title, round(1000 * distance) FROM test_entry_distance ORDER BY ngramSearchCaseInsensitive(Title, 'YanDEX') as distance, Title; drop table if exists test_entry_distance; diff --git a/tests/queries/0_stateless/00976_ttl_with_old_parts.sql b/tests/queries/0_stateless/00976_ttl_with_old_parts.sql index 8473a69aedd..c224ca30a3c 100644 --- a/tests/queries/0_stateless/00976_ttl_with_old_parts.sql +++ b/tests/queries/0_stateless/00976_ttl_with_old_parts.sql @@ -13,6 +13,6 @@ alter table ttl modify ttl d + interval 1 day; select sleep(1) format Null; -- wait if very fast merge happen optimize table ttl partition 10 final; -select * from ttl order by d; +select * from ttl order by d, a; drop table if exists ttl; diff --git a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh index aee8a7727e5..8dbd10fc27b 100755 --- a/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh +++ b/tests/queries/0_stateless/00992_system_parts_race_condition_zookeeper_long.sh @@ -81,5 +81,4 @@ check_replication_consistency "alter_table" "count(), sum(a), sum(b), round(sum( $CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table0;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & $CLICKHOUSE_CLIENT -n -q "DROP TABLE alter_table1;" 2> >(grep -F -v 'is already started to be removing by another replica right now') & - wait diff --git a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh index 72d3cbaa2f1..6b1df1d45a0 100755 --- a/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh +++ b/tests/queries/0_stateless/00993_system_parts_race_condition_drop_zookeeper.sh @@ -107,4 +107,5 @@ check_replication_consistency "alter_table_" "count(), sum(a), sum(b), round(sum for i in {0..9}; do $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS alter_table_$i" 2>&1 | grep "was not completely removed from ZooKeeper" & done + wait diff --git a/tests/queries/0_stateless/01018_Distributed__shard_num.reference b/tests/queries/0_stateless/01018_Distributed__shard_num.reference index b2c8b77554b..46963a006ec 100644 --- a/tests/queries/0_stateless/01018_Distributed__shard_num.reference +++ b/tests/queries/0_stateless/01018_Distributed__shard_num.reference @@ -1,34 +1,94 @@ +-- { echoOn } + +-- remote(system.one) +SELECT 'remote(system.one)'; remote(system.one) +SELECT * FROM remote('127.0.0.1', system.one); +0 +SELECT * FROM remote('127.0.0.{1,2}', system.one); 0 0 -0 +SELECT _shard_num, * FROM remote('127.0.0.1', system.one); 1 0 +SELECT _shard_num, * FROM remote('127.0.0.{1,2}', system.one) order by _shard_num; 1 0 2 0 +SELECT _shard_num, * FROM remote('127.0.0.{1,2}', system.one) WHERE _shard_num = 1; 1 0 +-- dist_1 using test_shard_localhost +SELECT 'dist_1'; dist_1 -1 -1 10 -10 +SELECT _shard_num FROM dist_1 order by _shard_num; 1 1 +SELECT _shard_num FROM dist_1 order by _shard_num; +1 +1 +SELECT _shard_num, key FROM dist_1 order by _shard_num; 1 10 1 20 +SELECT key FROM dist_1; 10 20 +SELECT _shard_num FROM dist_1 order by _shard_num; +1 +1 +SELECT _shard_num, key FROM dist_1 order by _shard_num, key; +1 10 +1 20 +SELECT key FROM dist_1; +10 +20 +-- dist_2 using test_cluster_two_shards_localhost +SELECT 'dist_2'; dist_2 +SELECT _shard_num FROM dist_2 order by _shard_num; 1 2 +SELECT _shard_num FROM dist_2 order by _shard_num; +1 +2 +SELECT _shard_num, key FROM dist_2 order by _shard_num, key; 1 100 2 100 +SELECT key FROM dist_2; 100 100 +-- multiple _shard_num +SELECT 'remote(Distributed)'; remote(Distributed) +SELECT _shard_num, key FROM remote('127.0.0.1', currentDatabase(), dist_2) order by _shard_num, key; 1 100 -1 100 +2 100 +-- JOIN system.clusters +SELECT 'JOIN system.clusters'; JOIN system.clusters +SELECT a._shard_num, a.key, b.host_name, b.host_address IN ('::1', '127.0.0.1'), b.port +FROM (SELECT *, _shard_num FROM dist_1) a +JOIN system.clusters b +ON a._shard_num = b.shard_num +WHERE b.cluster = 'test_cluster_two_shards_localhost'; 1 10 localhost 1 9000 1 20 localhost 1 9000 +SELECT _shard_num, key, b.host_name, b.host_address IN ('::1', '127.0.0.1'), b.port +FROM dist_1 a +JOIN system.clusters b +ON _shard_num = b.shard_num +WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 403 } +SELECT 'Rewrite with alias'; +Rewrite with alias +SELECT a._shard_num, key FROM dist_1 a; +1 10 +1 20 +-- the same with JOIN, just in case +SELECT a._shard_num, a.key, b.host_name, b.host_address IN ('::1', '127.0.0.1'), b.port +FROM dist_1 a +JOIN system.clusters b +ON a._shard_num = b.shard_num +WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 47; } +SELECT 'dist_3'; dist_3 +SELECT * FROM dist_3; 100 foo +SELECT _shard_num, * FROM dist_3 order by _shard_num; foo 100 foo diff --git a/tests/queries/0_stateless/01018_Distributed__shard_num.sql b/tests/queries/0_stateless/01018_Distributed__shard_num.sql index be2df8b664f..d3f4e1ac527 100644 --- a/tests/queries/0_stateless/01018_Distributed__shard_num.sql +++ b/tests/queries/0_stateless/01018_Distributed__shard_num.sql @@ -3,6 +3,28 @@ -- make the order static SET max_threads = 1; +DROP TABLE IF EXISTS mem1; +DROP TABLE IF EXISTS mem2; +DROP TABLE IF EXISTS mem3; +DROP TABLE IF EXISTS dist_1; +DROP TABLE IF EXISTS dist_2; +DROP TABLE IF EXISTS dist_3; + +CREATE TABLE mem1 (key Int) Engine=Memory(); +INSERT INTO mem1 VALUES (10); +CREATE TABLE dist_1 AS mem1 Engine=Distributed(test_shard_localhost, currentDatabase(), mem1); +INSERT INTO dist_1 VALUES (20); + +CREATE TABLE mem2 (key Int) Engine=Memory(); +INSERT INTO mem2 VALUES (100); +CREATE TABLE dist_2 AS mem2 Engine=Distributed(test_cluster_two_shards_localhost, currentDatabase(), mem2); + +CREATE TABLE mem3 (key Int, _shard_num String) Engine=Memory(); +INSERT INTO mem3 VALUES (100, 'foo'); +CREATE TABLE dist_3 AS mem3 Engine=Distributed(test_shard_localhost, currentDatabase(), mem3); + +-- { echoOn } + -- remote(system.one) SELECT 'remote(system.one)'; SELECT * FROM remote('127.0.0.1', system.one); @@ -13,27 +35,20 @@ SELECT _shard_num, * FROM remote('127.0.0.{1,2}', system.one) WHERE _shard_num = -- dist_1 using test_shard_localhost SELECT 'dist_1'; -CREATE TABLE mem1 (key Int) Engine=Memory(); -CREATE TABLE dist_1 AS mem1 Engine=Distributed(test_shard_localhost, currentDatabase(), mem1); SELECT _shard_num FROM dist_1 order by _shard_num; -INSERT INTO mem1 VALUES (10); SELECT _shard_num FROM dist_1 order by _shard_num; SELECT _shard_num, key FROM dist_1 order by _shard_num; SELECT key FROM dist_1; -INSERT INTO dist_1 VALUES (20); SELECT _shard_num FROM dist_1 order by _shard_num; SELECT _shard_num, key FROM dist_1 order by _shard_num, key; SELECT key FROM dist_1; -- dist_2 using test_cluster_two_shards_localhost SELECT 'dist_2'; -CREATE TABLE mem2 (key Int) Engine=Memory(); -CREATE TABLE dist_2 AS mem2 Engine=Distributed(test_cluster_two_shards_localhost, currentDatabase(), mem2); SELECT _shard_num FROM dist_2 order by _shard_num; -INSERT INTO mem2 VALUES (100); SELECT _shard_num FROM dist_2 order by _shard_num; SELECT _shard_num, key FROM dist_2 order by _shard_num, key; SELECT key FROM dist_2; @@ -57,8 +72,8 @@ JOIN system.clusters b ON _shard_num = b.shard_num WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 403 } --- rewrite does not work with aliases, hence Missing columns (47) -SELECT a._shard_num, key FROM dist_1 a; -- { serverError 47; } +SELECT 'Rewrite with alias'; +SELECT a._shard_num, key FROM dist_1 a; -- the same with JOIN, just in case SELECT a._shard_num, a.key, b.host_name, b.host_address IN ('::1', '127.0.0.1'), b.port FROM dist_1 a @@ -67,8 +82,5 @@ ON a._shard_num = b.shard_num WHERE b.cluster = 'test_cluster_two_shards_localhost'; -- { serverError 47; } SELECT 'dist_3'; -CREATE TABLE mem3 (key Int, _shard_num String) Engine=Memory(); -CREATE TABLE dist_3 AS mem3 Engine=Distributed(test_shard_localhost, currentDatabase(), mem3); -INSERT INTO mem3 VALUES (100, 'foo'); SELECT * FROM dist_3; SELECT _shard_num, * FROM dist_3 order by _shard_num; diff --git a/tests/queries/0_stateless/01030_limit_by_with_ties_error.sh b/tests/queries/0_stateless/01030_limit_by_with_ties_error.sh index 2f2b17072c3..711a015f044 100755 --- a/tests/queries/0_stateless/01030_limit_by_with_ties_error.sh +++ b/tests/queries/0_stateless/01030_limit_by_with_ties_error.sh @@ -6,10 +6,36 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) CLICKHOUSE_CLIENT=$(echo ${CLICKHOUSE_CLIENT} | sed 's/'"--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}"'/--send_logs_level=none/g') -$CLICKHOUSE_CLIENT --query="SELECT * FROM (SELECT number % 5 AS a, count() AS b, c FROM numbers(10) ARRAY JOIN [1,2] AS c GROUP BY a,c) AS table ORDER BY a LIMIT 3 WITH TIES BY a" 2>&1 | grep -q "Code: 498." && echo 'OK' || echo 'FAIL' ||: +$CLICKHOUSE_CLIENT --query=""" + SELECT * FROM (SELECT number % 5 AS a, count() AS b, c FROM numbers(10) + ARRAY JOIN [1,2] AS c GROUP BY a,c) AS table + ORDER BY a LIMIT 3 WITH TIES BY a""" 2>&1 | grep -q "Code: 498." && echo 'OK' || echo 'FAIL' ||: -$CLICKHOUSE_CLIENT --query="SELECT * FROM VALUES('Phrase String, Payload String', ('hello', 'x'), ('world', 'x'), ('hello', 'z'), ('upyachka', 'a'), ('test', 'b'), ('foo', 'c'), ('bar', 'd')) ORDER BY Payload LIMIT 1 WITH TIES BY Phrase LIMIT 5;" 2>&1 | grep -q "Code: 498." && echo 'OK' || echo 'FAIL' ||: +$CLICKHOUSE_CLIENT --query=""" + SELECT * FROM VALUES('Phrase String, Payload String', + ('hello', 'x'), ('world', 'x'), ('hello', 'z'), + ('upyachka', 'a'), ('test', 'b'), ('foo', 'c'), + ('bar', 'd')) + ORDER BY Payload LIMIT 1 WITH TIES BY Phrase LIMIT 5;""" 2>&1 | grep -q "Code: 498." && echo 'OK' || echo 'FAIL' ||: -$CLICKHOUSE_CLIENT --query="SELECT * FROM VALUES('Phrase String, Payload String', ('hello', 'x'), ('world', 'x'), ('hello', 'z'), ('upyachka', 'a'), ('test', 'b'), ('foo', 'c'), ('bar', 'd')) ORDER BY Payload LIMIT 1 BY Phrase LIMIT 5 WITH TIES" +$CLICKHOUSE_CLIENT --query=""" + SELECT * FROM + ( + SELECT * FROM VALUES('Phrase String, Payload String', + ('hello', 'x'), ('world', 'x'), ('hello', 'z'), + ('upyachka', 'a'), ('test', 'b'), ('foo', 'c'), + ('bar', 'd')) + ORDER BY Payload LIMIT 1 BY Phrase LIMIT 5 WITH TIES + ) ORDER BY Payload, Phrase + """ -$CLICKHOUSE_CLIENT --query="SELECT TOP 5 WITH TIES * FROM VALUES('Phrase String, Payload String', ('hello', 'x'), ('world', 'x'), ('hello', 'z'), ('upyachka', 'a'), ('test', 'b'), ('foo', 'c'), ('bar', 'd')) ORDER BY Payload LIMIT 1 BY Phrase" +$CLICKHOUSE_CLIENT --query=""" + SELECT * FROM + ( + SELECT TOP 5 WITH TIES * FROM VALUES('Phrase String, Payload String', + ('hello', 'x'), ('world', 'x'), ('hello', 'z'), + ('upyachka', 'a'), ('test', 'b'), ('foo', 'c'), + ('bar', 'd')) + ORDER BY Payload LIMIT 1 BY Phrase + ) ORDER BY Payload, Phrase + """ diff --git a/tests/queries/0_stateless/01031_new_any_join.reference b/tests/queries/0_stateless/01031_new_any_join.reference index 1fd9a5352e3..7b08703e422 100644 --- a/tests/queries/0_stateless/01031_new_any_join.reference +++ b/tests/queries/0_stateless/01031_new_any_join.reference @@ -7,10 +7,7 @@ any left any left (rev) 0 5 b6 2 a3 2 b1 -2 a3 2 b2 4 a5 4 b3 -4 a5 4 b4 -4 a5 4 b5 any inner 2 a3 2 b1 4 a5 4 b3 @@ -20,10 +17,7 @@ any inner (rev) any right 0 5 b6 2 a3 2 b1 -2 a3 2 b2 4 a5 4 b3 -4 a5 4 b4 -4 a5 4 b5 any right (rev) 0 a1 0 1 a2 0 diff --git a/tests/queries/0_stateless/01031_new_any_join.sql b/tests/queries/0_stateless/01031_new_any_join.sql index de86d8eebc5..2f2a8b2ad1d 100644 --- a/tests/queries/0_stateless/01031_new_any_join.sql +++ b/tests/queries/0_stateless/01031_new_any_join.sql @@ -5,7 +5,7 @@ CREATE TABLE t1 (x UInt32, s String) engine = Memory; CREATE TABLE t2 (x UInt32, s String) engine = Memory; INSERT INTO t1 (x, s) VALUES (0, 'a1'), (1, 'a2'), (2, 'a3'), (3, 'a4'), (4, 'a5'); -INSERT INTO t2 (x, s) VALUES (2, 'b1'), (2, 'b2'), (4, 'b3'), (4, 'b4'), (4, 'b5'), (5, 'b6'); +INSERT INTO t2 (x, s) VALUES (2, 'b1'), (4, 'b3'), (5, 'b6'); SET join_use_nulls = 0; SET any_join_distinct_right_table_keys = 0; diff --git a/tests/queries/0_stateless/01031_pmj_new_any_semi_join.reference b/tests/queries/0_stateless/01031_pmj_new_any_semi_join.reference index 78763e63f3f..b48b20942c6 100644 --- a/tests/queries/0_stateless/01031_pmj_new_any_semi_join.reference +++ b/tests/queries/0_stateless/01031_pmj_new_any_semi_join.reference @@ -3,45 +3,36 @@ any left 1 a2 0 2 a3 2 b1 3 a4 0 -4 a5 4 b3 +4 a5 4 b2 any left (rev) -0 5 b6 +0 5 b4 2 a3 2 b1 -2 a3 2 b2 -4 a5 4 b3 -4 a5 4 b4 -4 a5 4 b5 +4 a5 4 b2 any inner 2 a3 2 b1 -4 a5 4 b3 +4 a5 4 b2 any inner (rev) 2 a3 2 b1 -4 a5 4 b3 +4 a5 4 b2 any right -0 5 b6 +0 5 b4 2 a3 2 b1 -2 a3 2 b2 -4 a5 4 b3 -4 a5 4 b4 -4 a5 4 b5 +4 a5 4 b2 any right (rev) 0 a1 0 1 a2 0 2 a3 2 b1 3 a4 0 -4 a5 4 b3 +4 a5 4 b2 semi left 2 a3 2 b1 -4 a5 4 b3 +4 a5 4 b2 semi right 2 a3 2 b1 -2 a3 2 b2 -4 a5 4 b3 -4 a5 4 b4 -4 a5 4 b5 +4 a5 4 b2 anti left 0 a1 0 1 a2 1 3 a4 3 anti right -0 5 b6 +0 5 b4 diff --git a/tests/queries/0_stateless/01031_pmj_new_any_semi_join.sql b/tests/queries/0_stateless/01031_pmj_new_any_semi_join.sql index 4c306828bb8..0a593b3cfa9 100644 --- a/tests/queries/0_stateless/01031_pmj_new_any_semi_join.sql +++ b/tests/queries/0_stateless/01031_pmj_new_any_semi_join.sql @@ -5,7 +5,7 @@ CREATE TABLE t1 (x UInt32, s String) engine = Memory; CREATE TABLE t2 (x UInt32, s String) engine = Memory; INSERT INTO t1 (x, s) VALUES (0, 'a1'), (1, 'a2'), (2, 'a3'), (3, 'a4'), (4, 'a5'); -INSERT INTO t2 (x, s) VALUES (2, 'b1'), (2, 'b2'), (4, 'b3'), (4, 'b4'), (4, 'b5'), (5, 'b6'); +INSERT INTO t2 (x, s) VALUES (2, 'b1'), (4, 'b2'), (5, 'b4'); SET join_algorithm = 'prefer_partial_merge'; SET join_use_nulls = 0; diff --git a/tests/queries/0_stateless/01031_semi_anti_join.sql b/tests/queries/0_stateless/01031_semi_anti_join.sql index 19ea219563a..388b3d2fe4c 100644 --- a/tests/queries/0_stateless/01031_semi_anti_join.sql +++ b/tests/queries/0_stateless/01031_semi_anti_join.sql @@ -10,16 +10,16 @@ INSERT INTO t2 (x, s) VALUES (2, 'b1'), (2, 'b2'), (4, 'b3'), (4, 'b4'), (4, 'b5 SET join_use_nulls = 0; SELECT 'semi left'; -SELECT t1.*, t2.* FROM t1 SEMI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x; +SELECT t1.*, t2.* FROM t1 SEMI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s; SELECT 'semi right'; -SELECT t1.*, t2.* FROM t1 SEMI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x; +SELECT t1.*, t2.* FROM t1 SEMI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s; SELECT 'anti left'; -SELECT t1.*, t2.* FROM t1 ANTI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x; +SELECT t1.*, t2.* FROM t1 ANTI LEFT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s; SELECT 'anti right'; -SELECT t1.*, t2.* FROM t1 ANTI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x; +SELECT t1.*, t2.* FROM t1 ANTI RIGHT JOIN t2 USING(x) ORDER BY t1.x, t2.x, t1.s, t2.s; DROP TABLE t1; DROP TABLE t2; diff --git a/tests/queries/0_stateless/01069_database_memory.reference b/tests/queries/0_stateless/01069_database_memory.reference index e7486d57276..15e2bec3355 100644 --- a/tests/queries/0_stateless/01069_database_memory.reference +++ b/tests/queries/0_stateless/01069_database_memory.reference @@ -1,4 +1,4 @@ -CREATE DATABASE memory_01069\nENGINE = Memory() +CREATE DATABASE memory_01069\nENGINE = Memory 1 2 3 diff --git a/tests/queries/0_stateless/01070_mutations_with_dependencies.sql b/tests/queries/0_stateless/01070_mutations_with_dependencies.sql index 6f297e25b95..506fd23904f 100644 --- a/tests/queries/0_stateless/01070_mutations_with_dependencies.sql +++ b/tests/queries/0_stateless/01070_mutations_with_dependencies.sql @@ -1,4 +1,4 @@ --- Tags: no-parallel +-- Tags: no-parallel, no-s3-storage drop table if exists ttl; set mutations_sync = 2; diff --git a/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql b/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql index 2d6007c678c..3a05e4507a2 100644 --- a/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql +++ b/tests/queries/0_stateless/01078_merge_tree_read_one_thread.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage +-- Output slightly different plan drop table if exists t; create table t (a Int, b Int) engine = MergeTree order by (a, b) settings index_granularity = 400; diff --git a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh index fbeb18251a6..06d6ef6a94b 100755 --- a/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh +++ b/tests/queries/0_stateless/01079_parallel_alter_add_drop_column_zookeeper.sh @@ -80,7 +80,6 @@ timeout $TIMEOUT bash -c insert_thread 2> /dev/null & timeout $TIMEOUT bash -c insert_thread 2> /dev/null & timeout $TIMEOUT bash -c insert_thread 2> /dev/null & - wait echo "Finishing alters" diff --git a/tests/queries/0_stateless/01086_odbc_roundtrip.sh b/tests/queries/0_stateless/01086_odbc_roundtrip.sh index 8e59bfd7f4d..705746032f8 100755 --- a/tests/queries/0_stateless/01086_odbc_roundtrip.sh +++ b/tests/queries/0_stateless/01086_odbc_roundtrip.sh @@ -17,7 +17,7 @@ ${CLICKHOUSE_CLIENT} --query "select count() > 1 as ok from (select * from odbc( ${CLICKHOUSE_CLIENT} --query "CREATE TABLE t (x UInt8, y Float32, z String) ENGINE = Memory" ${CLICKHOUSE_CLIENT} --query "INSERT INTO t VALUES (1,0.1,'a я'),(2,0.2,'b ą'),(3,0.3,'c d')" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (ANSI)}','$CLICKHOUSE_DATABASE','t') ORDER BY x" -${CLICKHOUSE_CLIENT} --query "SELECT * FROM odbc('DSN={ClickHouse DSN (Unicode)}','$CLICKHOUSE_DATABASE','t') ORDER BY x" +${CLICKHOUSE_CLIENT} --query "SELECT x, y, z FROM odbc('DSN={ClickHouse DSN (ANSI)}','$CLICKHOUSE_DATABASE','t') ORDER BY x" +${CLICKHOUSE_CLIENT} --query "SELECT x, y, z FROM odbc('DSN={ClickHouse DSN (Unicode)}','$CLICKHOUSE_DATABASE','t') ORDER BY x" ${CLICKHOUSE_CLIENT} --query "DROP TABLE t" diff --git a/tests/queries/0_stateless/01142_with_ties_and_aliases.sql b/tests/queries/0_stateless/01142_with_ties_and_aliases.sql index f086cb9d907..de4a9281a08 100644 --- a/tests/queries/0_stateless/01142_with_ties_and_aliases.sql +++ b/tests/queries/0_stateless/01142_with_ties_and_aliases.sql @@ -1,12 +1,12 @@ -select number, intDiv(number,5) value from numbers(20) order by value limit 3 with ties; +select * from (select number, intDiv(number,5) value from numbers(20) order by value limit 3 with ties) ORDER BY number, value; drop table if exists wt; create table wt (a Int, b Int) engine = Memory; insert into wt select 0, number from numbers(5); select 1 from wt order by a limit 3 with ties; -select b from wt order by a limit 3 with ties; -with a * 2 as c select a, b from wt order by c limit 3 with ties; -select a * 2 as c, b from wt order by c limit 3 with ties; +select b from (select b from wt order by a limit 3 with ties) order by b; +select * from (select * from (with a * 2 as c select a, b from wt order by c limit 3 with ties) order by a, b); +select * from (select * from (select a * 2 as c, b from wt order by c limit 3 with ties) order by c, b); drop table if exists wt; diff --git a/tests/queries/0_stateless/01144_multiple_joins_rewriter_v2_and_lambdas.reference b/tests/queries/0_stateless/01144_multiple_joins_rewriter_v2_and_lambdas.reference index 0bf24c68939..dc0207a7b9f 100644 --- a/tests/queries/0_stateless/01144_multiple_joins_rewriter_v2_and_lambdas.reference +++ b/tests/queries/0_stateless/01144_multiple_joins_rewriter_v2_and_lambdas.reference @@ -1,3 +1,3 @@ [1] test query -[0,0] [0,0] [2,1] test query +[1,2] [3,4] [2,1] test query [] [] [] [] diff --git a/tests/queries/0_stateless/01154_move_partition_long.sh b/tests/queries/0_stateless/01154_move_partition_long.sh index 46cbaea5d44..6b0b0773cb6 100755 --- a/tests/queries/0_stateless/01154_move_partition_long.sh +++ b/tests/queries/0_stateless/01154_move_partition_long.sh @@ -126,4 +126,5 @@ for ((i=0; i<16; i++)) do $CLICKHOUSE_CLIENT -q "DROP TABLE dst_$i" 2>&1| grep -Fv "is already started to be removing" & $CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS src_$i" 2>&1| grep -Fv "is already started to be removing" & done + wait diff --git a/tests/queries/0_stateless/01193_metadata_loading.sh b/tests/queries/0_stateless/01193_metadata_loading.sh index f15032c6929..1604de6004a 100755 --- a/tests/queries/0_stateless/01193_metadata_loading.sh +++ b/tests/queries/0_stateless/01193_metadata_loading.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-parallel, no-fasttest +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-parallel, no-fasttest, no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql index de9c2df7f08..ff4918c0810 100644 --- a/tests/queries/0_stateless/01200_mutations_memory_consumption.sql +++ b/tests/queries/0_stateless/01200_mutations_memory_consumption.sql @@ -1,4 +1,4 @@ --- Tags: no-debug, no-parallel, long +-- Tags: no-debug, no-parallel, long, no-s3-storage DROP TABLE IF EXISTS table_with_single_pk; diff --git a/tests/queries/0_stateless/01221_system_settings.sql b/tests/queries/0_stateless/01221_system_settings.sql index 226be55503d..fcffd6c45fe 100644 --- a/tests/queries/0_stateless/01221_system_settings.sql +++ b/tests/queries/0_stateless/01221_system_settings.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage select * from system.settings where name = 'send_timeout'; select * from system.merge_tree_settings order by length(description) limit 1; diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index 86ba859fb0e..06bd6ab04e4 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -49,13 +49,13 @@ CREATE TABLE [] TABLE CREATE CREATE VIEW [] VIEW CREATE CREATE DICTIONARY [] DICTIONARY CREATE CREATE TEMPORARY TABLE [] GLOBAL CREATE -CREATE FUNCTION [] DATABASE CREATE +CREATE FUNCTION [] GLOBAL CREATE CREATE [] \N ALL DROP DATABASE [] DATABASE DROP DROP TABLE [] TABLE DROP DROP VIEW [] VIEW DROP DROP DICTIONARY [] DICTIONARY DROP -DROP FUNCTION [] DATABASE DROP +DROP FUNCTION [] GLOBAL DROP DROP [] \N ALL TRUNCATE ['TRUNCATE TABLE'] TABLE ALL OPTIMIZE ['OPTIMIZE TABLE'] TABLE ALL @@ -68,9 +68,9 @@ CREATE ROLE [] GLOBAL ACCESS MANAGEMENT ALTER ROLE [] GLOBAL ACCESS MANAGEMENT DROP ROLE [] GLOBAL ACCESS MANAGEMENT ROLE ADMIN [] GLOBAL ACCESS MANAGEMENT -CREATE ROW POLICY ['CREATE POLICY'] GLOBAL ACCESS MANAGEMENT -ALTER ROW POLICY ['ALTER POLICY'] GLOBAL ACCESS MANAGEMENT -DROP ROW POLICY ['DROP POLICY'] GLOBAL ACCESS MANAGEMENT +CREATE ROW POLICY ['CREATE POLICY'] TABLE ACCESS MANAGEMENT +ALTER ROW POLICY ['ALTER POLICY'] TABLE ACCESS MANAGEMENT +DROP ROW POLICY ['DROP POLICY'] TABLE ACCESS MANAGEMENT CREATE QUOTA [] GLOBAL ACCESS MANAGEMENT ALTER QUOTA [] GLOBAL ACCESS MANAGEMENT DROP QUOTA [] GLOBAL ACCESS MANAGEMENT @@ -79,7 +79,7 @@ ALTER SETTINGS PROFILE ['ALTER PROFILE'] GLOBAL ACCESS MANAGEMENT DROP SETTINGS PROFILE ['DROP PROFILE'] GLOBAL ACCESS MANAGEMENT SHOW USERS ['SHOW CREATE USER'] GLOBAL SHOW ACCESS SHOW ROLES ['SHOW CREATE ROLE'] GLOBAL SHOW ACCESS -SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY'] GLOBAL SHOW ACCESS +SHOW ROW POLICIES ['SHOW POLICIES','SHOW CREATE ROW POLICY','SHOW CREATE POLICY'] TABLE SHOW ACCESS SHOW QUOTAS ['SHOW CREATE QUOTA'] GLOBAL SHOW ACCESS SHOW SETTINGS PROFILES ['SHOW PROFILES','SHOW CREATE SETTINGS PROFILE','SHOW CREATE PROFILE'] GLOBAL SHOW ACCESS SHOW ACCESS [] \N ACCESS MANAGEMENT @@ -118,6 +118,7 @@ SYSTEM THREAD FUZZER ['SYSTEM START THREAD FUZZER','SYSTEM STOP THREAD FUZZER',' SYSTEM [] \N ALL dictGet ['dictHas','dictGetHierarchy','dictIsIn'] DICTIONARY ALL addressToLine [] GLOBAL INTROSPECTION +addressToLineWithInlines [] GLOBAL INTROSPECTION addressToSymbol [] GLOBAL INTROSPECTION demangle [] GLOBAL INTROSPECTION INTROSPECTION ['INTROSPECTION FUNCTIONS'] \N ALL diff --git a/tests/queries/0_stateless/01277_fromUnixTimestamp64.reference b/tests/queries/0_stateless/01277_fromUnixTimestamp64.reference index 610041de31e..8c951058ea6 100644 --- a/tests/queries/0_stateless/01277_fromUnixTimestamp64.reference +++ b/tests/queries/0_stateless/01277_fromUnixTimestamp64.reference @@ -3,3 +3,7 @@ UTC 1234567891011 2009-02-13 23:31:31.011 1970-01-15 06:56:07.891011 1970-01-01 Asia/Makassar 1234567891011 2009-02-14 07:31:31.011 1970-01-15 14:56:07.891011 1970-01-01 08:20:34.567891011 DateTime64(9, \'Asia/Makassar\') non-const column 1234567891011 2009-02-13 23:31:31.011 1970-01-15 06:56:07.891011 1970-01-01 00:20:34.567891011 +upper range bound +9904447342 2283-11-10 19:22:22.123 2283-11-10 19:22:22.123456 1925-01-01 00:00:00.586094827 +lower range bound +-1420066799 1925-01-01 01:00:01.123 1925-01-01 01:00:01.123456 1925-01-01 01:00:01.123456789 diff --git a/tests/queries/0_stateless/01277_fromUnixTimestamp64.sql b/tests/queries/0_stateless/01277_fromUnixTimestamp64.sql index 4f1497763e1..e76a4db7a27 100644 --- a/tests/queries/0_stateless/01277_fromUnixTimestamp64.sql +++ b/tests/queries/0_stateless/01277_fromUnixTimestamp64.sql @@ -42,4 +42,30 @@ SELECT i64, fromUnixTimestamp64Milli(i64, tz), fromUnixTimestamp64Micro(i64, tz), - fromUnixTimestamp64Nano(i64, tz) as dt64; \ No newline at end of file + fromUnixTimestamp64Nano(i64, tz) as dt64; + +SELECT 'upper range bound'; +WITH + 9904447342 AS timestamp, + CAST(9904447342123 AS Int64) AS milli, + CAST(9904447342123456 AS Int64) AS micro, + CAST(9904447342123456789 AS Int64) AS nano, + 'UTC' AS tz +SELECT + timestamp, + fromUnixTimestamp64Milli(milli, tz), + fromUnixTimestamp64Micro(micro, tz), + fromUnixTimestamp64Nano(nano, tz); + +SELECT 'lower range bound'; +WITH + -1420066799 AS timestamp, + CAST(-1420066799123 AS Int64) AS milli, + CAST(-1420066799123456 AS Int64) AS micro, + CAST(-1420066799123456789 AS Int64) AS nano, + 'UTC' AS tz +SELECT + timestamp, + fromUnixTimestamp64Milli(milli, tz), + fromUnixTimestamp64Micro(micro, tz), + fromUnixTimestamp64Nano(nano, tz); \ No newline at end of file diff --git a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect index e4442047c87..5057ec44e8a 100755 --- a/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect +++ b/tests/queries/0_stateless/01293_client_interactive_vertical_multiline.expect @@ -1,7 +1,7 @@ #!/usr/bin/expect -f log_user 0 -set timeout 60 +set timeout 10 match_max 100000 expect_after { @@ -11,6 +11,9 @@ expect_after { timeout { exit 1 } } +# useful debugging configuration +# exp_internal 1 + set basedir [file dirname $argv0] spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion" expect ":) " @@ -41,7 +44,7 @@ expect ":) " send -- "" expect eof -spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --multiline" +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT --disable_suggestion --highlight 0 --multiline" expect ":) " send -- "SELECT 1;\r" diff --git a/tests/queries/0_stateless/01318_map_populate_series.reference b/tests/queries/0_stateless/01318_map_populate_series.reference index 2d83844c8e1..65e6f8e462c 100644 --- a/tests/queries/0_stateless/01318_map_populate_series.reference +++ b/tests/queries/0_stateless/01318_map_populate_series.reference @@ -13,6 +13,11 @@ ([1,2,3,4,5,6,7,8,9,10],[1,0,2,0,0,0,0,0,0,0]) ([1,2,3,4,5,6,7,8,9,10],[1,0,0,2,0,0,0,0,0,0]) ([1,2,3,4,5,6,7,8,9,10],[1,0,0,0,2,0,0,0,0,0]) +([1,2,3,4,5,6,7,8,9,10],[1,0,0,0,0,0,0,0,0,0]) +([1,2,3,4,5,6,7,8,9,10],[1,2,0,0,0,0,0,0,0,0]) +([1,2,3,4,5,6,7,8,9,10],[1,0,2,0,0,0,0,0,0,0]) +([1,2,3,4,5,6,7,8,9,10],[1,0,0,2,0,0,0,0,0,0]) +([1,2,3,4,5,6,7,8,9,10],[1,0,0,0,2,0,0,0,0,0]) ([1,2],[1,0]) ([1,2,3],[1,2,0]) ([1,2,3,4],[1,0,2,0]) diff --git a/tests/queries/0_stateless/01318_map_populate_series.sql b/tests/queries/0_stateless/01318_map_populate_series.sql index e52571182fe..f7fa8c81e8c 100644 --- a/tests/queries/0_stateless/01318_map_populate_series.sql +++ b/tests/queries/0_stateless/01318_map_populate_series.sql @@ -4,7 +4,7 @@ create table map_test engine=TinyLog() as (select (number + 1) as n, ([1, number select mapPopulateSeries(map.1, map.2) from map_test; select mapPopulateSeries(map.1, map.2, toUInt64(3)) from map_test; select mapPopulateSeries(map.1, map.2, toUInt64(10)) from map_test; -select mapPopulateSeries(map.1, map.2, 1000) from map_test; -- { serverError 43 } +select mapPopulateSeries(map.1, map.2, 10) from map_test; select mapPopulateSeries(map.1, map.2, n) from map_test; select mapPopulateSeries(map.1, [11,22]) from map_test; select mapPopulateSeries([3, 4], map.2) from map_test; @@ -31,6 +31,6 @@ select mapPopulateSeries([toInt64(-10), 2], [toInt64(1), 1], toInt64(-5)) as res -- empty select mapPopulateSeries(cast([], 'Array(UInt8)'), cast([], 'Array(UInt8)'), 5); -select mapPopulateSeries(['1', '2'], [1,1]) as res, toTypeName(res); -- { serverError 43 } -select mapPopulateSeries([1, 2, 3], [1,1]) as res, toTypeName(res); -- { serverError 42 } -select mapPopulateSeries([1, 2], [1,1,1]) as res, toTypeName(res); -- { serverError 42 } +select mapPopulateSeries(['1', '2'], [1, 1]) as res, toTypeName(res); -- { serverError 43 } +select mapPopulateSeries([1, 2, 3], [1, 1]) as res, toTypeName(res); -- { serverError 36 } +select mapPopulateSeries([1, 2], [1, 1, 1]) as res, toTypeName(res); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01322_monotonous_order_by_with_different_variables.reference b/tests/queries/0_stateless/01322_monotonous_order_by_with_different_variables.reference index cf2935a40bf..186e6565ffe 100644 --- a/tests/queries/0_stateless/01322_monotonous_order_by_with_different_variables.reference +++ b/tests/queries/0_stateless/01322_monotonous_order_by_with_different_variables.reference @@ -11,8 +11,8 @@ 2 1 3 3 1 4 3 -2 5 4 2 2 4 +2 5 4 2 1 4 3 1 3 3 @@ -27,6 +27,6 @@ 2 1 3 3 1 4 3 -2 5 4 2 2 4 +2 5 4 2 diff --git a/tests/queries/0_stateless/01322_monotonous_order_by_with_different_variables.sql b/tests/queries/0_stateless/01322_monotonous_order_by_with_different_variables.sql index 6fda42cbed1..87f0f462ab9 100644 --- a/tests/queries/0_stateless/01322_monotonous_order_by_with_different_variables.sql +++ b/tests/queries/0_stateless/01322_monotonous_order_by_with_different_variables.sql @@ -7,7 +7,7 @@ SELECT * FROM test ORDER BY toFloat32(x), -y, -z DESC; SELECT * FROM test ORDER BY toFloat32(x), -(-y), -z DESC; SELECT max(x) as k FROM test ORDER BY k; SELECT roundToExp2(x) as k FROM test GROUP BY k ORDER BY k; -SELECT roundToExp2(x) as k, y, z FROM test WHERE k >= 1 ORDER BY k; +SELECT roundToExp2(x) as k, y, z FROM test WHERE k >= 1 ORDER BY k, y, z; SELECT max(x) as k FROM test HAVING k > 0 ORDER BY k; SET optimize_monotonous_functions_in_order_by = 0; @@ -15,7 +15,7 @@ SELECT * FROM test ORDER BY toFloat32(x), -y, -z DESC; SELECT * FROM test ORDER BY toFloat32(x), -(-y), -z DESC; SELECT max(x) as k FROM test ORDER BY k; SELECT roundToExp2(x) as k From test GROUP BY k ORDER BY k; -SELECT roundToExp2(x) as k, y, z FROM test WHERE k >= 1 ORDER BY k; +SELECT roundToExp2(x) as k, y, z FROM test WHERE k >= 1 ORDER BY k, y, z; SELECT max(x) as k FROM test HAVING k > 0 ORDER BY k; DROP TABLE test; diff --git a/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql b/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql index c8c49ea8c58..614629351ef 100644 --- a/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql +++ b/tests/queries/0_stateless/01343_min_bytes_to_use_mmap_io.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS test_01343; CREATE TABLE test_01343 (x String) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_01343 VALUES ('Hello, world'); diff --git a/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql b/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql index 7805ecea392..2e5ec563641 100644 --- a/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql +++ b/tests/queries/0_stateless/01344_min_bytes_to_use_mmap_io_index.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS test_01344; CREATE TABLE test_01344 (x String, INDEX idx (x) TYPE set(10) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS min_bytes_for_wide_part = 0; INSERT INTO test_01344 VALUES ('Hello, world'); diff --git a/tests/queries/0_stateless/01375_output_format_tsv_csv_with_names.sh b/tests/queries/0_stateless/01375_output_format_tsv_csv_with_names.sh index 69f3ab1c9a8..462b9078abc 100755 --- a/tests/queries/0_stateless/01375_output_format_tsv_csv_with_names.sh +++ b/tests/queries/0_stateless/01375_output_format_tsv_csv_with_names.sh @@ -5,7 +5,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) . "$CURDIR"/../shell_config.sh opts=( - --input-format CSV -q 'SELECT number FROM numbers(2)' ) diff --git a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh index fbbf74bbba2..95d99449837 100755 --- a/tests/queries/0_stateless/01455_opentelemetry_distributed.sh +++ b/tests/queries/0_stateless/01455_opentelemetry_distributed.sh @@ -20,7 +20,7 @@ select attribute['db.statement'] as query, attribute['clickhouse.tracestate'] as tracestate, 1 as sorted_by_start_time from system.opentelemetry_span_log - where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + where trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' order by start_time_us ; @@ -31,7 +31,7 @@ select attribute['db.statement'] as query, attribute['clickhouse.tracestate'] as tracestate, 1 as sorted_by_finish_time from system.opentelemetry_span_log - where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + where trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' order by finish_time_us ; @@ -43,7 +43,7 @@ select count(*) "'"'"total spans"'"'", uniqExactIf(parent_span_id, parent_span_id != 0) "'"'"unique non-zero parent spans"'"'" from system.opentelemetry_span_log - where trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + where trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' ; @@ -56,7 +56,7 @@ select count(*) "'"'"initial query spans with proper parent"'"'" mapValues(attribute) as attribute_value) o join system.query_log on query_id = o.attribute_value where - trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and current_database = currentDatabase() and operation_name = 'query' and parent_span_id = reinterpretAsUInt64(unhex('73')) @@ -71,7 +71,7 @@ select uniqExact(value) "'"'"unique non-empty tracestate values"'"'" from system.opentelemetry_span_log array join mapKeys(attribute) as name, mapValues(attribute) as value where - trace_id = reinterpretAsUUID(reverse(unhex('$trace_id'))) + trace_id = UUIDNumToString(toFixedString(unhex('$trace_id'), 16)) and operation_name = 'query' and name = 'clickhouse.tracestate' and length(value) > 0 diff --git a/tests/queries/0_stateless/01475_read_subcolumns.sql b/tests/queries/0_stateless/01475_read_subcolumns.sql index fa849c889b4..4724bec9eff 100644 --- a/tests/queries/0_stateless/01475_read_subcolumns.sql +++ b/tests/queries/0_stateless/01475_read_subcolumns.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + SET use_uncompressed_cache = 0; SELECT '====array===='; diff --git a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference index b8b8fae2830..5abc312652d 100644 --- a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference +++ b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.reference @@ -5,7 +5,7 @@ 0 1 0 -2 +1 0 4 6 diff --git a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql index ecf0b791a49..e4e2e3dd76a 100644 --- a/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql +++ b/tests/queries/0_stateless/01505_trivial_count_with_partition_predicate.sql @@ -31,7 +31,7 @@ select count() from test_tuple where toDate(p) > '2020-09-01'; -- optimized select count() from test_tuple where toDate(p) > '2020-09-01' and i = 1; -- optimized -select count() from test_tuple where i > 1; +select count() from test_tuple where i > 2; -- optimized select count() from test_tuple where i < 1; -- non-optimized diff --git a/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh b/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh index f3cdb70a074..55b6110918b 100755 --- a/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh +++ b/tests/queries/0_stateless/01509_parallel_quorum_and_merge_long.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash -# Tags: long, no-replicated-database +# Tags: long, no-replicated-database, no-s3-storage # Tag no-replicated-database: Fails due to additional replicas or shards +# Tag no-s3-storage: Merge assigned to replica 2, but replication queues are stopped for it + set -e diff --git a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference index 422a076b0cb..19a1b0f2ec0 100644 --- a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference +++ b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.reference @@ -6,5 +6,5 @@ 3 3 1 1 2 1 -3 4 3 3 +3 4 diff --git a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql index fce7dd753d2..d02a2af6666 100644 --- a/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql +++ b/tests/queries/0_stateless/01525_select_with_offset_fetch_clause.sql @@ -3,6 +3,6 @@ SELECT number FROM numbers(10) ORDER BY number DESC OFFSET 2 ROWS FETCH NEXT 3 R DROP TABLE IF EXISTS test_fetch; CREATE TABLE test_fetch(a Int32, b Int32) Engine = Memory; INSERT INTO test_fetch VALUES(1, 1), (2, 1), (3, 4), (3, 3), (5, 4), (0, 6), (5, 7); -SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS ONLY; -SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS WITH TIES; +SELECT * FROM (SELECT * FROM test_fetch ORDER BY a, b OFFSET 1 ROW FETCH FIRST 3 ROWS ONLY) ORDER BY a, b; +SELECT * FROM (SELECT * FROM test_fetch ORDER BY a OFFSET 1 ROW FETCH FIRST 3 ROWS WITH TIES) ORDER BY a, b; DROP TABLE test_fetch; diff --git a/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql b/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql index 8df6d0b314e..cf06af0113d 100644 --- a/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql +++ b/tests/queries/0_stateless/01532_execute_merges_on_single_replica_long.sql @@ -1,4 +1,4 @@ --- Tags: long, replica, no-replicated-database, no-parallel +-- Tags: long, replica, no-replicated-database, no-parallel, no-s3-storage -- Tag no-replicated-database: Fails due to additional replicas or shards -- Tag no-parallel: static zk path diff --git a/tests/queries/0_stateless/01533_collate_in_nullable.sql b/tests/queries/0_stateless/01533_collate_in_nullable.sql index 9e54581bc54..9664a8efdb3 100644 --- a/tests/queries/0_stateless/01533_collate_in_nullable.sql +++ b/tests/queries/0_stateless/01533_collate_in_nullable.sql @@ -7,9 +7,9 @@ CREATE TABLE test_collate (x UInt32, s Nullable(String)) ENGINE=Memory(); INSERT INTO test_collate VALUES (1, 'Ё'), (1, 'ё'), (1, 'а'), (1, null), (2, 'А'), (2, 'я'), (2, 'Я'), (2, null); SELECT 'Order by without collate'; -SELECT * FROM test_collate ORDER BY s; +SELECT * FROM test_collate ORDER BY s, x; SELECT 'Order by with collate'; -SELECT * FROM test_collate ORDER BY s COLLATE 'ru'; +SELECT * FROM test_collate ORDER BY s COLLATE 'ru', x; SELECT 'Order by tuple without collate'; SELECT * FROM test_collate ORDER BY x, s; diff --git a/tests/queries/0_stateless/01533_multiple_nested.sql b/tests/queries/0_stateless/01533_multiple_nested.sql index 40aef40eaca..03724ce0b46 100644 --- a/tests/queries/0_stateless/01533_multiple_nested.sql +++ b/tests/queries/0_stateless/01533_multiple_nested.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage +-- Temporary supressed DROP TABLE IF EXISTS nested; SET flatten_nested = 0; diff --git a/tests/queries/0_stateless/01543_collate_in_tuple.sql b/tests/queries/0_stateless/01543_collate_in_tuple.sql index 222f7762b32..e50b5e5223d 100644 --- a/tests/queries/0_stateless/01543_collate_in_tuple.sql +++ b/tests/queries/0_stateless/01543_collate_in_tuple.sql @@ -12,19 +12,19 @@ INSERT INTO collate_test1 VALUES (1, (1, 'Ё')), (1, (1, 'ё')), (1, (1, 'а')), INSERT INTO collate_test2 VALUES (1, (1, 'Ё')), (1, (1, 'ё')), (1, (1, 'а')), (2, (2, 'А')), (2, (1, 'я')), (2, (2, 'Я')), (1, (2, null)), (1, (3, 'я')), (1, (1, null)), (2, (2, null)); INSERT INTO collate_test3 VALUES (1, (1, (1, ['Ё']))), (1, (2, (1, ['ё']))), (1, (1, (2, ['а']))), (2, (1, (1, ['А']))), (2, (2, (1, ['я']))), (2, (1, (1, ['Я']))), (1, (2, (1, ['ё','а']))), (1, (1, (2, ['ё', 'я']))), (2, (1, (1, ['ё', 'а', 'а']))); -SELECT * FROM collate_test1 ORDER BY s COLLATE 'ru'; +SELECT * FROM collate_test1 ORDER BY s COLLATE 'ru', x; SELECT ''; SELECT * FROM collate_test1 ORDER BY x, s COLLATE 'ru'; SELECT ''; -SELECT * FROM collate_test2 ORDER BY s COLLATE 'ru'; +SELECT * FROM collate_test2 ORDER BY s COLLATE 'ru', x; SELECT ''; SELECT * FROM collate_test2 ORDER BY x, s COLLATE 'ru'; SELECT ''; -SELECT * FROM collate_test3 ORDER BY s COLLATE 'ru'; +SELECT * FROM collate_test3 ORDER BY s COLLATE 'ru', x; SELECT ''; SELECT * FROM collate_test3 ORDER BY x, s COLLATE 'ru'; diff --git a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql index a71c9f9a714..1d21d861e20 100644 --- a/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql +++ b/tests/queries/0_stateless/01551_mergetree_read_in_order_spread.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + DROP TABLE IF EXISTS data_01551; CREATE TABLE data_01551 diff --git a/tests/queries/0_stateless/01558_ttest.reference b/tests/queries/0_stateless/01558_ttest.reference index 473960c0e7c..5cbc86038c3 100644 --- a/tests/queries/0_stateless/01558_ttest.reference +++ b/tests/queries/0_stateless/01558_ttest.reference @@ -8,7 +8,9 @@ -0.5028215369187079 0.6152361677171103 14.971190998235835 5.898143508382202e-44 14.971190998235837 0 +7.650530175770567 9.960200184229425 -2.610898982580138 0.00916587538237954 -2.610898982580134 0.0091658753823834 -28.740781574102936 7.667329672103986e-133 -28.74078157410298 0 +-9.625938422388245 -8.395483817611758 diff --git a/tests/queries/0_stateless/01558_ttest.sql b/tests/queries/0_stateless/01558_ttest.sql index 7a66bba9d2a..94a48c38fcc 100644 --- a/tests/queries/0_stateless/01558_ttest.sql +++ b/tests/queries/0_stateless/01558_ttest.sql @@ -30,6 +30,7 @@ CREATE TABLE welch_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO welch_ttest VALUES (4.82025, 0), (6.13896, 0), (15.20277, 0), (14.15351, 0), (7.21338, 0), (8.55506, 0), (13.80816, 0), (11.28411, 0), (7.4612, 0), (7.43759, 0), (12.9832, 0), (-5.74783, 0), (12.47114, 0), (15.14223, 0), (3.40603, 0), (9.27323, 0), (7.88547, 0), (8.56456, 0), (4.59731, 0), (7.91213, 0), (7.33894, 0), (21.74811, 0), (11.92111, 0), (0.18828, 0), (10.47314, 0), (20.37396, 0), (11.04991, 0), (13.30083, 0), (14.28065, 0), (2.86942, 0), (24.96072, 0), (14.20164, 0), (18.28769, 0), (10.50949, 0), (9.22273, 0), (11.77608, 0), (8.56872, 0), (13.74535, 0), (11.65209, 0), (12.51894, 0), (17.76256, 0), (13.52122, 0), (8.70796, 0), (6.04749, 0), (16.33064, 0), (8.35636, 0), (14.03496, 0), (11.05834, 0), (14.49261, 0), (2.59383, 0), (8.01022, 0), (4.05458, 0), (13.26384, 0), (14.62058, 0), (10.52489, 0), (8.46357, 0), (6.4147, 0), (9.70071, 0), (12.47581, 0), (4.38333, 0), (17.54172, 0), (10.12109, 0), (7.73186, 0), (14.0279, 0), (11.6621, 0), (17.47045, 0), (15.50223, 0), (15.46034, 0), (13.39964, 0), (14.98025, 0), (15.87912, 0), (17.67374, 0), (9.64073, 0), (12.84904, 0), (7.70278, 0), (13.03156, 0), (9.04512, 0), (15.97014, 0), (8.96389, 0), (11.48009, 0), (9.71153, 0), (13.00084, 0), (12.39803, 0), (13.08188, 0), (5.82244, 0), (10.81871, 0), (8.2539, 0), (7.52114, 0), (9.11488, 0), (8.37482, 0), (14.48652, 0), (11.42152, 0), (16.03111, 0), (13.14057, 0), (-2.26351, 0), (15.50394, 0), (14.88603, 0), (13.37257, 0), (11.84026, 0), (7.66558, 0), (6.24584, 0), (3.6312, 0), (2.7018, 0), (5.63656, 0), (5.82643, 0), (10.06745, 0), (-0.5831, 0), (14.84202, 0), (9.5524, 0), (19.71713, 0), (14.23109, 0), (8.69105, 0), (5.33742, 0), (7.30372, 0), (7.93342, 0), (15.20884, 0), (7.53839, 0), (13.45311, 0), (11.04473, 0), (10.76673, 0), (15.44145, 0), (14.06596, 0), (9.14873, 0), (12.88372, 0), (8.74994, 0), (10.53263, 0), (16.16694, 0), (8.37197, 0), (3.43739, 0), (4.72799, 0), (9.08802, 0), (11.2531, 0), (5.16115, 0), (10.20895, 0), (18.70884, 0), (15.88924, 0), (3.38758, 0), (6.46449, 0), (10.21088, 0), (14.08458, 0), (15.74508, 0), (19.31896, 0), (13.19641, 0), (11.95409, 0), (10.70718, 0), (1.05245, 0), (10.04772, 0), (17.01369, 0), (10.2286, 0), (19.58323, 0), (7.02892, 0), (4.16866, 0), (8.94326, 0), (4.99854, 0), (8.88352, 0), (18.65422, 0), (17.32328, 0), (9.33492, 0), (14.94788, 0), (8.05863, 0), (14.6737, 0), (10.93801, 0), (0.54036, 0), (-0.34242, 0), (5.89076, 0), (3.15189, 0), (1.94421, 0), (6.38698, 0), (10.50654, 0), (8.95362, 0), (6.23711, 0), (11.75359, 0), (12.42155, 0), (-1.55472, 0), (4.6688, 0), (10.48087, 0), (11.74615, 0), (9.26822, 0), (7.55517, 0), (12.76005, 0), (16.47102, 0), (11.31297, 0), (14.37437, 0), (2.38799, 0), (6.44577, 0), (5.07471, 0), (11.55123, 0), (7.76795, 0), (10.60116, 0), (14.40885, 0), (11.58158, 0), (8.81648, 0), (12.92299, 0), (11.26939, 0), (17.95014, 0), (2.95002, 0), (17.41959, 0), (11.12455, 0), (8.78541, 0), (14.36413, 0), (12.98554, 0), (12.58505, 0), (15.49789, 0), (11.70999, 0), (0.65596, 0), (11.08202, 0), (14.75752, 0), (6.84385, 0), (9.27245, 0), (13.78243, 0), (17.4863, 0), (4.01777, 0), (11.82861, 0), (13.86551, 0), (6.16591, 0), (8.71589, 0), (16.77195, 0), (17.23243, 0), (-2.12941, 0), (5.66629, 0), (12.45153, 0), (1.63971, 0), (13.84031, 0), (4.6144, 0), (5.26169, 0), (9.27769, 0), (9.14288, 0), (9.71953, 0), (9.38446, 0), (1.64788, 0), (11.72922, 0), (13.68926, 0), (9.42952, 0), (12.05574, 0), (9.09148, 0), (5.32273, 0), (20.25258, 0), (10.14599, 0), (10.82156, 0), (5.75736, 0), (7.13567, 0), (9.29746, 0), (5.1618, 0), (10.076, 0), (21.65669, 0), (13.35486, 0), (6.79957, 0), (8.76243, 0), (14.59294, 0), (16.90609, 0), (10.50337, 0), (-0.07923, 0), (13.51648, 0), (12.0676, 0), (0.86482, 0), (9.03563, 0), (5.38751, 0), (17.16866, 0), (2.78702, 0), (11.15548, 0), (12.30843, 0), (8.04897, 0), (9.95814, 0), (11.29308, 0), (14.13032, 0), (21.05877, 0), (3.57386, 0), (7.96631, 0), (3.30484, 0), (18.61856, 0), (16.35184, 0), (7.65236, 0), (18.02895, 0), (9.79458, 0), (16.7274, 0), (8.84453, 0), (13.05709, 0), (10.91447, 0), (8.40171, 0), (16.95211, 0), (11.82194, 0), (19.87978, 0), (12.88455, 0), (-0.00947, 0), (12.28109, 0), (6.96462, 0), (13.75282, 0), (14.39141, 0), (11.07193, 0), (12.88039, 0), (11.38253, 0), (21.02707, 0), (7.51955, 0), (6.31984, 0), (15.6543, 0), (14.80315, 0), (8.38024, 0), (21.7516, 0), (14.31336, 0), (15.04703, 0), (5.73787, 0), (13.16911, 0), (12.40695, 0), (9.88968, 0), (8.46703, 0), (8.70637, 0), (8.03551, 0), (5.9757, 0), (12.22951, 0), (3.14736, 0), (10.51266, 0), (18.593, 0), (10.82213, 0), (7.14216, 0), (6.81154, 0), (-0.6486, 0), (20.56136, 0), (11.35367, 0), (11.38205, 0), (17.14, 0), (14.91215, 0), (15.50207, 0), (5.93162, 0), (3.74869, 0), (14.11532, 0), (7.38954, 0), (5.45764, 0), (18.33733, 0), (9.91923, 0), (2.38991, 0), (14.16756, 0), (2.39791, 0), (6.92586, 0), (5.32474, 0), (2.28812, 0), (5.71718, 0), (5.84197, 0), (2.76206, 0), (19.05928, 0), (11.51788, 0), (6.56648, 0), (3.35735, 0), (7.55948, 0), (19.99908, 0), (13.00634, 0), (18.36886, 0), (11.14675, 0), (16.72931, 0), (12.50106, 0), (6.00605, 0), (23.06653, 0), (5.39694, 0), (9.53167, 0), (12.76944, 0), (7.20604, 0), (13.25391, 0), (13.7341, 0), (10.85292, 0), (-7.75835, 0), (10.29728, 0), (13.70099, 0), (10.17959, 0), (9.98399, 0), (12.69389, 0), (-0.28848, 0), (-2.18319, 0), (13.36378, 0), (10.09232, 0), (5.49489, 0), (5.46156, 0), (0.94225, 0), (12.79205, 0), (10.09593, 0), (6.06218, 0), (0.89463, 0), (11.88986, 0), (10.79733, 0), (1.51371, 0), (2.20967, 0), (15.45732, 0), (16.5262, 0), (5.99724, 0), (8.3613, 0), (15.68183, 0), (15.32117, 0), (14.15674, 0), (6.64553, 0), (4.20777, 0), (-0.10521, 0), (-0.88169, 0), (1.85913, 0), (9.73673, 0), (0.30926, 0), (6.17559, 0), (11.76602, 0), (5.68385, 0), (14.57088, 0), (12.81509, 0), (9.85682, 0), (12.06376, 0), (6.08874, 0), (11.63921, 0), (14.86722, 0), (10.41035, 0), (2.93794, 0), (12.21841, 0), (0.23804, 0), (3.14845, 0), (7.29748, 0), (3.06134, 0), (13.77684, 0), (16.21992, 0), (5.33511, 0), (9.68959, 0), (9.44169, 0), (18.08012, 0), (4.04224, 0), (8.77918, 0), (10.18324, 0), (9.38914, 0), (11.76995, 0), (14.19963, 0), (6.88817, 0), (16.56123, 0), (15.39885, 0), (5.21241, 0), (4.44408, 0), (17.87587, 0), (12.53337, 0), (13.60916, 0), (6.60104, 0), (7.35453, 0), (18.61572, 0), (6.10437, 0), (13.08682, 0), (12.15404, 0), (4.90789, 0), (2.13353, 0), (12.49593, 0), (11.93056, 0), (13.29408, 0), (5.70038, 0), (8.40271, 0), (5.19456, 0), (-5.51028, 0), (14.0329, 0), (10.38365, 0), (6.56812, 0), (4.21129, 0), (9.7157, 0), (9.88553, 0), (13.45346, 0), (4.97752, 0), (12.77595, 0), (8.56465, 0), (4.27703, 0), (18.12502, 0), (12.45735, 0), (12.42912, 0), (12.08125, 0), (10.85779, 0), (4.36013, 0), (11.85062, 0), (8.47776, 0), (9.60822, 0), (11.3069, 0), (14.25525, 0), (1.55168, 0), (14.57782, 0), (7.84786, 0), (9.87774, 0), (14.75575, 0), (3.68774, 0), (9.37667, 0), (20.28676, 0), (12.10027, 0), (8.01819, 0), (18.78158, 0), (20.85402, 0), (18.98069, 0), (16.1429, 0), (9.24047, 0), (14.12487, 0), (10.18841, 0), (-3.04478, 0), (5.7552, 0), (9.30376, 0), (11.42837, 0), (6.02364, 0), (8.86984, 0), (10.91177, 0), (10.04418, 0), (18.10774, 0), (7.49384, 0), (9.11556, 0), (9.7051, 0), (5.23268, 0), (9.04647, 0), (8.81547, 0), (2.65098, 0), (-2.69857, 1), (15.80943, 1), (7.31555, 1), (3.96517, 1), (4.77809, 1), (9.6472, 1), (-26.41717, 1), (-10.85635, 1), (-1.4376, 1), (-0.96308, 1), (2.84315, 1), (5.79467, 1), (-3.06091, 1), (-14.62902, 1), (22.08022, 1), (-2.11982, 1), (-4.84824, 1), (-10.50447, 1), (2.4891, 1), (9.90324, 1), (-22.66866, 1), (-0.97103, 1), (-16.57608, 1), (-3.78749, 1), (25.84511, 1), (5.30797, 1), (-18.19466, 1), (11.72708, 1), (0.2891, 1), (-9.83474, 1), (6.69942, 1), (18.09604, 1), (18.52651, 1), (1.38201, 1), (7.64615, 1), (17.66598, 1), (-2.44141, 1), (-9.01598, 1), (27.69142, 1), (4.06946, 1), (-15.0077, 1), (-10.49648, 1), (-4.88322, 1), (-25.09805, 1), (-4.64024, 1), (20.94434, 1), (24.12126, 1), (-14.10962, 1), (10.6512, 1), (14.50687, 1), (-19.88081, 1), (-11.55271, 1), (13.16921, 1), (16.63864, 1), (-24.08114, 1), (-9.09949, 1), (-10.54702, 1), (0.20813, 1), (8.19066, 1), (-2.70523, 1), (-0.23954, 1), (7.19398, 1), (-7.1618, 1), (-7.44322, 1), (-17.92031, 1), (-1.58146, 1), (9.18338, 1), (3.25838, 1), (-14.30234, 1), (1.84695, 1), (31.13794, 1), (-0.85067, 1), (19.02787, 1), (-3.09594, 1), (13.45584, 1), (-5.48104, 1), (-22.74928, 1), (-8.03697, 1), (17.31143, 1), (-16.65231, 1), (-18.58713, 1), (-16.52641, 1), (14.95261, 1), (12.56762, 1), (15.00188, 1), (1.85858, 1), (2.1926, 1), (-2.4095, 1), (21.56873, 1), (3.35509, 1), (-4.98672, 1), (35.08603, 1), (-10.01602, 1), (-3.85153, 1), (-6.81974, 1), (19.56525, 1), (-9.35488, 1), (0.24268, 1), (-3.51488, 1), (-0.37066, 1), (24.20888, 1), (-11.73537, 1), (0.01282, 1), (0.03963, 1), (-9.65589, 1), (-0.37429, 1), (5.61255, 1), (0.49984, 1), (-10.15066, 1), (-14.54314, 1), (16.56889, 1), (-7.73873, 1), (-3.76422, 1), (1.40722, 1), (2.28818, 1), (-13.12643, 1), (5.17082, 1), (4.79089, 1), (-17.42643, 1), (8.72548, 1), (-3.70285, 1), (16.77893, 1), (13.382, 1), (19.98418, 1), (0.00483, 1), (-4.75951, 1), (2.35391, 1), (21.65809, 1), (-9.2714, 1), (-18.38253, 1), (7.23097, 1), (14.97927, 1), (-4.02197, 1), (-29.8189, 1), (-12.8554, 1), (-7.60124, 1), (-14.90158, 1), (-3.31486, 1), (31.38144, 1), (-8.61288, 1), (15.31895, 1), (-10.19488, 1), (13.796, 1), (-0.32912, 1), (-0.0684, 1), (-30.06834, 1), (24.93912, 1), (-3.26506, 1), (-8.29751, 1), (-5.39189, 1), (-25.08603, 1), (-1.45318, 1), (16.72724, 1), (-3.38467, 1), (-26.00478, 1), (7.28369, 1), (16.96226, 1), (16.5858, 1), (10.46583, 1), (3.84345, 1), (-2.99382, 1), (1.42078, 1), (-11.0123, 1), (2.09909, 1), (1.21064, 1), (15.36079, 1), (-21.61349, 1), (22.7726, 1), (10.50512, 1), (-6.95825, 1), (9.20036, 1), (15.66902, 1), (3.28098, 1), (-9.05692, 1), (0.32882, 1), (-1.64934, 1), (-4.81406, 1), (-5.06006, 1), (19.97493, 1), (2.88646, 1), (-0.34552, 1), (7.55186, 1), (-22.96115, 1), (31.29166, 1), (6.18798, 1), (-2.52715, 1), (-11.58799, 1), (14.13596, 1), (13.45069, 1), (12.15179, 1), (3.44491, 1), (-8.78006, 1), (18.32087, 1), (11.91757, 1), (-2.00179, 1), (10.88411, 1), (9.09327, 1), (6.62484, 1), (8.87178, 1), (11.52254, 1), (-14.15988, 1), (-17.19515, 1), (14.03089, 1), (-2.4095, 1), (-16.83575, 1), (2.71469, 1), (4.84351, 1), (-1.17651, 1), (-3.37529, 1), (-19.92137, 1), (4.48952, 1), (-12.4906, 1), (-5.65277, 1), (8.50819, 1), (-19.61261, 1), (12.54156, 1), (11.06784, 1), (-12.59285, 1), (3.43683, 1), (-3.00325, 1), (12.49082, 1), (7.20955, 1), (17.6547, 1), (15.8619, 1), (24.3048, 1), (-8.05434, 1), (-6.06901, 1), (-15.69515, 1), (-11.13917, 1), (-3.90757, 1), (-2.57038, 1), (5.14065, 1), (17.8497, 1), (-8.64665, 1), (-18.68331, 1), (5.8567, 1), (-20.93884, 1), (4.40583, 1), (14.35985, 1), (4.18134, 1), (4.3635, 1), (9.35428, 1), (2.8908, 1), (16.01017, 1), (-1.48499, 1), (-9.97949, 1), (1.03055, 1), (-2.79697, 1), (6.85977, 1), (4.73213, 1), (2.7815, 1), (-2.46866, 1), (18.39425, 1), (-0.80378, 1), (-0.22982, 1), (-16.11608, 1), (3.0862, 1), (3.20779, 1), (10.50146, 1), (-0.21305, 1), (11.21012, 1), (-0.99825, 1), (18.39633, 1), (-3.39003, 1), (-0.64411, 1), (-1.39932, 1), (15.45319, 1), (-0.66044, 1), (-15.2223, 1), (-34.39907, 1), (-3.57836, 1), (16.82828, 1), (1.66624, 1), (15.43475, 1), (8.17776, 1), (5.50486, 1), (10.43082, 1), (-6.63332, 1), (2.28008, 1), (16.37203, 1), (5.16313, 1), (-8.85281, 1), (13.26692, 1), (-7.46842, 1), (8.43091, 1), (-13.18172, 1), (-0.72401, 1), (22.3881, 1), (10.65448, 1), (2.81289, 1), (10.92405, 1), (-8.95358, 1), (19.80653, 1), (-12.86527, 1), (5.38826, 1), (-6.83501, 1), (-15.7647, 1), (-27.67412, 1), (8.6499, 1), (-4.89542, 1), (16.76167, 1), (12.84284, 1), (-17.27324, 1), (-4.18726, 1), (-14.62366, 1), (-5.49863, 1), (-16.22846, 1), (10.60329, 1), (6.46781, 1), (1.70458, 1), (10.77448, 1), (0.8463, 1), (13.0482, 1), (-4.36264, 1), (3.22647, 1), (2.38828, 1), (6.7946, 1), (-0.25254, 1), (1.2497, 1), (1.6544, 1), (4.1019, 1), (11.27839, 1), (-5.04127, 1), (18.11674, 1), (0.51231, 1), (-0.51029, 1), (13.52556, 1), (16.10171, 1), (5.68197, 1), (-2.85904, 1), (-8.89167, 1), (6.24489, 1), (10.85319, 1), (-0.39816, 1), (3.87079, 1), (-3.1867, 1), (1.55322, 1), (16.86779, 1), (-14.60321, 1), (-1.81952, 1), (-3.11624, 1), (1.24193, 1), (10.18179, 1), (4.69796, 1), (0.69032, 1), (11.7723, 1), (7.62896, 1), (9.89741, 1), (9.11484, 1), (-3.84676, 1), (-0.4777, 1), (0.95958, 1), (-7.95056, 1), (-10.97474, 1), (-6.54861, 1), (34.74933, 1), (27.39463, 1), (4.18299, 1), (6.02476, 1), (-1.99397, 1), (1.26478, 1), (23.37106, 1), (10.49682, 1), (-11.04354, 1), (-12.22284, 1), (-9.87635, 1), (28.90511, 1), (6.77613, 1), (0.55352, 1), (0.37031, 1), (7.1418, 1), (3.24897, 1), (-1.60918, 1), (3.1675, 1), (-17.97072, 1), (-5.61743, 1), (14.1422, 1), (14.87695, 1), (-4.65961, 1), (-0.99174, 1), (-2.96623, 1), (-9.02263, 1), (-17.2088, 1), (2.78608, 1), (6.74239, 1), (4.8524, 1), (7.46731, 1), (1.04894, 1), (-12.8023, 1), (-17.18188, 1), (-5.08801, 1), (22.13942, 1), (-0.36384, 1), (17.80564, 1), (7.67504, 1), (1.59779, 1), (4.10942, 1), (0.61074, 1), (-14.40767, 1), (10.59906, 1), (16.57017, 1), (-15.17526, 1), (-6.98549, 1), (-0.64548, 1), (3.23756, 1), (14.65504, 1), (4.583, 1), (12.72378, 1), (5.26547, 1), (0.81781, 1), (9.38273, 1), (10.37636, 1), (10.70325, 1), (-0.83043, 1), (-7.53149, 1), (-9.09147, 1), (-19.51381, 1), (-28.44508, 1), (6.44392, 1), (11.10201, 1), (-2.86184, 1), (8.30673, 1), (8.8797, 1), (10.68053, 1), (15.62919, 1), (8.00579, 1), (6.4651, 1), (-4.50029, 1), (18.04514, 1), (11.12996, 1), (-5.14007, 1), (9.43857, 1), (3.13476, 1), (4.9772, 1), (-17.45782, 1), (0.05552, 1), (-1.90283, 1), (2.67908, 1), (-2.62243, 1), (-3.22767, 1), (-8.70222, 1), (-23.11605, 1), (21.6757, 1), (12.70076, 1), (4.4322, 1), (11.69344, 1), (9.18052, 1), (-2.2549, 1), (-2.15615, 1), (20.29765, 1), (-0.29536, 1), (15.50109, 1), (8.79187, 1), (5.11533, 1), (-20.44436, 1), (-3.00909, 1), (-4.48291, 1), (21.84462, 1), (1.94225, 1), (-2.81908, 1), (17.19418, 1), (-9.33528, 1), (-0.17346, 1), (0.03958, 1), (-35.17786, 1), (8.36887, 1), (-9.02292, 1), (-10.98804, 1), (0.29335, 1), (4.29634, 1), (3.87718, 1), (-9.08532, 1), (7.13922, 1), (-7.62463, 1), (-10.5666, 1), (4.68165, 1), (-3.30172, 1), (13.04852, 1), (13.45616, 1), (2.41043, 1), (-0.36501, 1), (-15.67383, 1), (17.92217, 1), (8.42106, 1), (3.22063, 1), (-7.31753, 1), (21.99596, 1), (-36.8273, 1), (-20.46391, 1), (5.74179, 1), (-15.83178, 1), (14.90454, 1), (-8.84645, 1), (3.72036, 1), (4.6877, 1), (16.35418, 1), (3.15441, 1), (2.39907, 1), (-17.58664, 1), (-13.18269, 1); SELECT '14.971190998235835', '5.898143508382202e-44'; SELECT roundBankers(welchTTest(left, right).1, 16) as t_stat, roundBankers(welchTTest(left, right).2, 16) as p_value from welch_ttest; +SELECT roundBankers(welchTTest(0.95)(left, right).3, 16) as t_stat, roundBankers(welchTTest(0.95)(left, right).4, 16) as p_value from welch_ttest; DROP TABLE IF EXISTS welch_ttest; @@ -52,4 +53,5 @@ CREATE TABLE student_ttest (left Float64, right Float64) ENGINE = Memory; INSERT INTO student_ttest VALUES (4.52546, 0), (8.69444, 1), (3.73628, 0), (3.81414, 1), (-0.39478, 0), (12.38442, 1), (5.15633, 0), (8.9738, 1), (0.50539, 0), (9.19594, 1), (-5.34036, 0), (7.21009, 1), (0.19336, 0), (4.97743, 1), (8.35729, 0), (4.94756, 1), (6.95818, 0), (19.80911, 1), (-2.93812, 0), (13.75358, 1), (8.30807, 0), (16.56373, 1), (-3.3517, 0), (9.72882, 1), (4.16279, 0), (4.64509, 1), (-3.17231, 0), (17.76854, 1), (1.93545, 0), (4.80693, 1), (11.06606, 0), (8.79505, 1), (-4.22678, 0), (10.88868, 1), (-1.99975, 0), (6.21932, 1), (-4.51178, 0), (15.11614, 1), (-4.50711, 0), (13.24703, 1), (1.89786, 0), (14.76476, 1), (-6.19638, 0), (-0.6117, 1), (-3.70188, 0), (17.48993, 1), (5.01334, 0), (12.11847, 1), (1.79036, 0), (4.87439, 1), (2.14435, 0), (18.56479, 1), (3.0282, 0), (1.23712, 1), (2.35528, 0), (5.41596, 1), (-12.18535, 0), (4.54994, 1), (5.59709, 0), (11.37668, 1), (-12.92336, 0), (9.5982, 1), (-0.04281, 0), (6.59822, 1), (-0.16923, 0), (1.16703, 1), (0.88924, 0), (8.88418, 1), (-4.68414, 0), (10.95047, 1), (8.01099, 0), (5.52787, 1), (2.61686, 0), (-1.11647, 1), (-2.76895, 0), (14.49946, 1), (3.32165, 0), (3.27585, 1), (-0.85135, 0), (-0.42025, 1), (1.21368, 0), (6.37906, 1), (4.38673, 0), (2.5242, 1), (6.20964, 0), (8.1405, 1), (-1.23172, 0), (6.46732, 1), (4.65516, 0), (9.89332, 1), (-1.87143, 0), (10.4374, 1), (0.86429, 0), (-1.06465, 1), (2.51184, 0), (6.84902, 1), (-1.88822, 0), (10.96576, 1), (-1.61802, 0), (7.83319, 1), (1.93653, 0), (14.39823, 1), (-3.66631, 0), (7.02594, 1), (-1.05294, 0), (13.46629, 1), (-10.74718, 0), (10.39531, 1), (16.49295, 0), (11.27348, 1), (-7.65494, 0), (9.32187, 1), (-3.39303, 0), (12.32667, 1), (-4.89418, 0), (8.98905, 1), (3.2521, 0), (9.54757, 1), (0.05831, 0), (5.98325, 1), (-3.00409, 0), (3.47248, 1), (5.76702, 0), (9.26966, 1), (2.67674, 0), (5.77816, 1), (10.52623, 0), (6.32966, 1), (-0.54501, 0), (9.49313, 1), (-4.89835, 0), (6.21337, 1), (3.52457, 0), (10.00242, 1), (-0.0451, 0), (6.25167, 1), (-6.61226, 0), (15.64671, 1), (9.02391, 0), (2.78968, 1), (5.52571, 0), (6.55442, 1), (4.54352, 0), (3.68819, 1), (-3.8394, 0), (9.55934, 1), (-7.75295, 0), (4.166, 1), (5.91167, 0), (12.32471, 1), (1.38897, 0), (7.10969, 1), (6.24166, 0), (16.31723, 1), (5.58536, 0), (12.99482, 1), (4.7591, 0), (10.11585, 1), (-2.58336, 0), (10.29455, 1), (-1.91263, 0), (18.27524, 1), (3.31575, 0), (12.84435, 1), (5.3507, 0), (13.11954, 1), (-15.22081, 0), (12.84147, 1), (-0.84775, 0), (15.55658, 1), (-4.538, 0), (11.45329, 1), (6.71177, 0), (7.50912, 1), (0.52882, 0), (8.56226, 1), (2.0242, 0), (8.63104, 1), (5.69146, 0), (15.68026, 1), (4.63328, 0), (21.6361, 1), (0.22984, 0), (6.23925, 1), (-2.84052, 0), (8.65714, 1), (7.91867, 0), (9.9423, 1), (1.11001, 0), (12.28213, 1), (-0.11251, 0), (3.11279, 1), (-0.20905, 0), (13.58128, 1), (0.03287, 0), (16.51407, 1), (-1.59397, 0), (16.60476, 1), (-5.39405, 0), (12.02022, 1), (-7.1233, 0), (12.11035, 1), (4.51517, 0), (9.47832, 1), (-0.70967, 0), (6.40742, 1), (5.67299, 0), (8.87252, 1), (-0.33835, 0), (15.14265, 1), (-1.83047, 0), (2.23572, 1), (-0.62877, 0), (11.57144, 1), (-7.23148, 0), (18.87737, 1), (0.1802, 0), (12.1833, 1), (11.73325, 0), (11.17519, 1), (2.17603, 0), (16.80422, 1), (-0.11683, 0), (6.81423, 1), (-1.29102, 0), (12.12546, 1), (-0.23201, 0), (8.06153, 1), (-6.8643, 0), (10.97228, 1), (-6.85153, 0), (7.30596, 1), (-4.77163, 0), (15.44026, 1), (6.11721, 0), (8.00993, 1), (5.96406, 0), (12.60196, 1), (3.59135, 0), (13.96832, 1), (-0.60095, 0), (14.03207, 1), (3.11163, 0), (4.53758, 1), (-0.18831, 0), (8.08297, 1), (0.67657, 0), (4.90451, 1), (-3.16117, 0), (8.14253, 1), (0.26957, 0), (19.88605, 1), (2.18653, 0), (13.85254, 1), (-5.94611, 0), (23.01839, 1), (-4.39352, 0), (6.02084, 1), (-3.71525, 0), (9.60319, 1), (5.11103, 0), (1.90511, 1), (1.33998, 0), (10.35237, 1), (1.01629, 0), (16.27082, 1), (-3.36917, 0), (12.52379, 1), (-3.99661, 0), (11.37435, 1), (8.19336, 0), (13.61823, 1), (2.89168, 0), (15.77622, 1), (-11.10373, 0), (15.17254, 1), (11.68005, 0), (6.711, 1), (3.08282, 0), (4.74205, 1), (-6.81506, 0), (10.09812, 1), (-2.34587, 0), (6.61722, 1), (-2.68725, 0), (10.34164, 1), (0.3577, 0), (8.96602, 1), (-3.05682, 0), (12.32157, 1), (9.08062, 0), (11.75711, 1), (-0.77913, 0), (13.49499, 1), (10.35215, 0), (8.57713, 1), (6.82565, 0), (11.50313, 1), (-1.24674, 0), (1.13097, 1), (5.18822, 0), (7.83205, 1), (-3.70743, 0), (5.77957, 1), (1.40319, 0), (15.5519, 1), (5.89432, 0), (10.82676, 1), (1.43152, 0), (11.51218, 1), (6.70638, 0), (9.29779, 1), (9.76613, 0), (9.77021, 1), (4.27604, 0), (9.94114, 1), (-2.63141, 0), (15.54513, 1), (-7.8133, 0), (19.10736, 1), (-0.06668, 0), (15.04205, 1), (1.05391, 0), (9.03114, 1), (4.41797, 0), (24.0104, 1), (0.09337, 0), (9.94205, 1), (6.16075, 0), (2.5925, 1), (7.49413, 0), (8.82726, 1), (-3.52872, 0), (10.0209, 1), (-2.17126, 0), (8.1635, 1), (-3.87605, 0), (4.24074, 1), (3.26607, 0), (7.67291, 1), (-3.28045, 0), (5.21642, 1), (2.1429, 0), (11.2808, 1), (1.53386, 0), (6.88172, 1), (0.21169, 0), (5.98743, 1), (-0.63674, 0), (17.97249, 1), (5.84893, 0), (6.46323, 1), (-0.63498, 0), (15.37416, 1), (8.29526, 0), (2.89957, 1), (-1.08358, 0), (17.13044, 1), (-2.306, 0), (11.06355, 1), (2.86991, 0), (3.09625, 1), (-0.76074, 0), (-2.33019, 1), (5.49191, 0), (7.42675, 1), (1.82883, 0), (15.06792, 1), (-3.70497, 0), (8.81116, 1), (-0.53232, 0), (19.17446, 1), (-11.49722, 0), (18.77181, 1), (3.44877, 0), (14.06443, 1), (-1.8596, 0), (12.81241, 1), (-10.34851, 0), (2.72299, 1), (1.13093, 0), (18.67739, 1), (-10.93389, 0), (11.63275, 1), (-3.39703, 0), (2.23891, 1), (0.19749, 0), (13.01195, 1), (-3.68389, 0), (7.43402, 1), (-4.67863, 0), (8.14599, 1), (10.78916, 0), (16.65328, 1), (0.37675, 0), (1.362, 1), (3.98094, 0), (3.87957, 1), (-3.64775, 0), (11.16134, 1), (-4.8443, 0), (6.25357, 1), (1.102, 0), (4.21945, 1), (8.72112, 0), (12.50047, 1), (-1.47361, 0), (6.45486, 1), (6.24183, 0), (18.99924, 1), (6.83569, 0), (18.09508, 1), (-3.11684, 0), (13.59528, 1), (4.91306, 0), (3.39681, 1), (-0.03628, 0), (13.33157, 1), (5.1282, 0), (5.8945, 1), (-2.38558, 0), (5.61212, 1), (2.33351, 0), (8.41149, 1), (-0.97191, 0), (13.78608, 1), (-0.05588, 0), (6.08609, 1), (-4.70019, 0), (12.76962, 1), (-5.12371, 0), (3.26206, 1), (0.65606, 0), (0.25528, 1), (-0.11574, 0), (11.9083, 1), (4.4238, 0), (4.35071, 1), (6.93399, 0), (11.19855, 1), (3.68712, 0), (13.87404, 1), (-0.01187, 0), (6.87986, 1), (1.8332, 0), (8.32566, 1), (5.81322, 0), (22.51334, 1), (-4.04709, 0), (2.5226, 1), (-8.26397, 0), (16.84498, 1), (-2.11273, 0), (6.26108, 1), (5.28396, 0), (13.84824, 1), (0.73054, 0), (6.03262, 1), (6.43559, 0), (14.12668, 1), (4.35565, 0), (16.01939, 1), (-1.05545, 0), (8.19237, 1), (5.00087, 0), (18.01595, 1), (-2.72239, 0), (9.45609, 1), (7.32313, 0), (6.90459, 1), (2.11548, 0), (12.83115, 1), (-3.40953, 0), (10.603, 1), (6.97051, 0), (13.70439, 1), (-0.45567, 0), (6.1633, 1), (1.31699, 0), (4.1151, 1), (-1.49871, 0), (8.20499, 1), (7.14772, 0), (11.67903, 1), (0.79277, 0), (7.30851, 1), (6.9698, 0), (6.50941, 1), (2.08733, 0), (7.3949, 1), (-3.55962, 0), (12.80075, 1), (0.75601, 0), (5.62043, 1), (1.21, 0), (18.2542, 1), (-2.17877, 0), (17.9393, 1), (1.83206, 0), (16.4569, 1), (5.72463, 0), (8.78811, 1), (7.42257, 0), (4.85949, 1), (0.97829, 0), (-3.36394, 1), (7.54238, 0), (5.38683, 1), (9.91081, 0), (12.26083, 1), (-4.61743, 0), (10.27907, 1), (-4.40799, 0), (11.5144, 1), (9.99854, 0), (11.57335, 1), (8.53725, 0), (1.94203, 1), (3.2905, 0), (7.78228, 1), (0.38634, 0), (11.79385, 1), (-2.53374, 0), (10.18415, 1), (4.94758, 0), (14.67613, 1), (4.79624, 0), (4.70301, 1), (5.57664, 0), (12.72151, 1), (-6.44871, 0), (-3.35508, 1), (3.34431, 0), (17.63775, 1), (0.14209, 0), (2.53883, 1), (10.88431, 0), (14.01483, 1), (0.31846, 0), (12.4387, 1), (-0.54703, 0), (11.15408, 1), (-4.67791, 0), (7.74882, 1), (-5.68011, 0), (13.60956, 1), (-4.93362, 0), (7.81991, 1), (1.2271, 0), (10.90969, 1), (5.27512, 0), (8.19828, 1), (-3.84611, 0), (-1.18523, 1), (6.81706, 0), (0.5916, 1), (10.33033, 0), (0.35805, 1), (5.13979, 0), (12.98364, 1), (3.66534, 0), (11.38628, 1), (-2.07219, 0), (13.94644, 1), (10.65442, 0), (2.03781, 1), (-3.31751, 0), (10.74447, 1), (-1.82011, 0), (12.35656, 1), (-0.39886, 0), (7.08701, 1), (1.77052, 0), (2.69871, 1), (1.29049, 0), (19.66653, 1), (7.92344, 0), (7.88636, 1), (-2.92595, 0), (10.36916, 1), (-2.67107, 0), (1.632, 1), (5.64708, 0), (11.86081, 1), (0.34639, 0), (13.47602, 1), (-3.04356, 0), (6.60204, 1), (3.98828, 0), (7.01303, 1), (-1.36695, 0), (20.19992, 1), (-8.48462, 0), (18.88249, 1), (-4.04669, 0), (11.34367, 1), (9.84561, 0), (12.97305, 1), (-6.1537, 0), (9.5776, 1), (0.82433, 0), (17.91364, 1), (1.92449, 0), (18.3247, 1), (2.51288, 0), (9.9211, 1), (0.40965, 0), (7.14257, 1), (2.89183, 0), (6.59133, 1), (3.84347, 0), (12.35274, 1), (0.66829, 0), (10.57523, 1), (-3.45094, 0), (12.12859, 1), (1.3544, 0), (9.47177, 1), (-9.85456, 0), (0.60659, 1), (5.25689, 0), (4.72996, 1), (-5.26018, 0), (4.51121, 1), (-6.16912, 0), (13.28893, 1), (-1.77163, 0), (8.09014, 1), (3.96687, 0), (8.02511, 1), (0.70893, 0), (13.85406, 1), (-5.45342, 0), (1.75412, 1), (-3.89706, 0), (6.00641, 1), (3.11868, 0), (6.35554, 1), (4.41714, 0), (7.11293, 1), (7.64841, 0), (8.30442, 1), (0.00489, 0), (12.63024, 1), (3.2263, 0), (12.38966, 1), (-5.33042, 0), (7.6801, 1), (2.52189, 0), (11.33744, 1), (-7.40308, 0), (4.67713, 1), (0.67891, 0), (7.62276, 1), (2.49343, 0), (2.14478, 1), (5.43133, 0), (15.32988, 1), (-0.67541, 0), (1.52299, 1), (-0.60299, 0), (17.00017, 1), (-6.32903, 0), (8.29701, 1), (-3.44336, 0), (10.92961, 1), (-0.23963, 0), (6.78449, 1), (6.94686, 0), (7.02698, 1), (6.59442, 0), (11.51719, 1), (-4.18532, 0), (9.97926, 1), (-1.8228, 0), (7.44251, 1), (-0.29443, 0), (7.58541, 1), (2.99821, 0), (4.76058, 1), (2.51942, 0), (12.88959, 1), (-3.49176, 0), (9.974, 1), (-0.57979, 0), (17.03689, 1), (8.69471, 0), (11.14554, 1), (-1.19427, 0), (11.7392, 1), (-3.17119, 0), (11.50029, 1), (-2.99566, 0), (19.41759, 1), (-3.34493, 0), (9.65127, 1), (-2.33826, 0), (9.87673, 1), (-5.04164, 0), (14.13485, 1), (-0.48214, 0), (9.78034, 1), (7.45097, 0), (1.57826, 1), (3.04787, 0), (3.72091, 1), (2.92632, 0), (9.4054, 1), (1.39694, 0), (23.22816, 1), (4.38686, 0), (-0.12571, 1), (3.25753, 0), (6.97343, 1), (7.14218, 0), (10.09049, 1), (-4.04341, 0), (11.78393, 1), (-9.19352, 0), (3.01909, 1), (2.78473, 0), (16.09448, 1), (0.33331, 0), (6.25485, 1), (9.89238, 0), (7.13164, 1), (6.00566, 0), (7.75879, 1), (-1.7511, 0), (9.56834, 1), (4.77815, 0), (6.14824, 1), (5.07457, 0), (13.53454, 1), (2.56132, 0), (8.26364, 1), (2.38317, 0), (8.7095, 1), (-1.63486, 0), (10.61607, 1), (-1.46871, 0), (10.64418, 1), (-5.8681, 0), (23.9106, 1), (-2.96227, 0), (11.38978, 1), (-1.90638, 0), (11.4383, 1), (-13.3052, 0), (18.41498, 1), (-2.14705, 0), (3.70959, 1), (-9.62069, 0), (19.95918, 1), (2.29313, 0), (9.53847, 1), (0.22162, 0), (14.04957, 1), (-1.83956, 0), (13.70151, 1), (4.1853, 0), (5.45046, 1), (6.05965, 0), (10.95061, 1), (-0.23737, 0), (9.55156, 1), (6.07452, 0), (17.92345, 1), (4.34629, 0), (6.23976, 1), (4.02922, 0), (8.71029, 1), (3.62622, 0), (13.58736, 1), (-3.95825, 0), (8.78527, 1), (-1.63412, 0), (11.14213, 1), (-1.25727, 0), (12.23717, 1), (5.06323, 0), (16.44557, 1), (-0.66176, 0), (0.47144, 1), (2.36606, 0), (9.7198, 1), (-5.77792, 0), (13.50981, 1), (4.535, 0), (14.27806, 1), (1.02031, 0), (13.50793, 1), (4.49345, 0), (7.47381, 1), (-4.99791, 0), (11.07844, 1), (2.46716, 0), (9.89844, 1), (3.65471, 0), (21.48548, 1), (11.2283, 0), (6.92085, 1), (6.69743, 0), (4.44074, 1), (-5.60375, 0), (19.98074, 1), (0.28683, 0), (7.92826, 1), (-0.85737, 0), (16.6313, 1), (4.26726, 0), (17.17618, 1), (-3.4322, 0), (13.80807, 1), (-2.07039, 0), (5.37083, 1), (-2.26798, 0), (9.73962, 1), (-0.99818, 0), (10.66273, 1), (0.41335, 0), (8.90639, 1), (5.18124, 0), (12.24596, 1), (-5.01858, 0), (16.89203, 1), (2.05561, 0), (12.69184, 1), (-0.12117, 0), (15.59077, 1), (0.99471, 0), (6.94287, 1), (6.89979, 0), (-0.1801, 1), (-4.18527, 0), (3.25318, 1), (-6.35104, 0), (8.08804, 1), (3.89734, 0), (13.78384, 1), (-1.979, 0), (0.46434, 1), (3.15404, 0), (7.78224, 1), (3.52672, 0), (9.10987, 1), (2.48372, 0), (-0.89391, 1), (-6.13089, 0), (14.3696, 1), (2.2968, 0), (3.01763, 1), (-2.74324, 0), (8.03559, 1), (-0.12876, 0), (7.24609, 1), (-1.51135, 0), (11.86271, 1), (-3.92434, 0), (6.28196, 1), (-1.71254, 0), (8.9725, 1), (-1.25878, 0), (14.46114, 1), (2.03021, 0), (9.50216, 1), (4.31726, 0), (16.30413, 1), (-3.02908, 0), (1.02795, 1), (9.7093, 0), (1.88717, 1), (-3.36284, 0), (9.80106, 1), (6.70938, 0), (4.53487, 1), (0.42762, 0), (16.34543, 1), (5.04726, 0), (7.71098, 1), (2.78386, 0), (2.74639, 1), (6.83022, 0), (6.51875, 1), (-3.02109, 0), (10.42308, 1), (-0.65382, 0), (13.57901, 1), (-15.58675, 0), (0.52784, 1), (5.89746, 0), (4.4708, 1), (-4.11598, 0), (6.39619, 1), (-1.37208, 0), (14.57666, 1), (10.08082, 0), (2.71602, 1), (5.35686, 0), (12.53905, 1), (1.93331, 0), (11.4292, 1), (10.47444, 0), (12.44641, 1), (-2.36872, 0), (14.50894, 1), (6.50752, 0), (17.64374, 1), (2.54603, 0), (11.03218, 1), (-0.4332, 0), (9.82789, 1), (5.26572, 0), (10.11104, 1), (2.09016, 0), (2.16137, 1), (1.15513, 0), (10.24054, 1), (14.95941, 0), (12.86909, 1), (-3.85505, 0), (15.22845, 1), (-2.36239, 0), (5.05411, 1), (1.64338, 0), (10.84836, 1), (-4.25074, 0), (11.15717, 1), (7.29744, 0), (0.91782, 1), (-1.18964, 0), (13.29961, 1), (5.60612, 0), (15.11314, 1), (-3.77011, 0), (11.54004, 1), (6.67642, 0), (-0.94238, 1), (-0.06862, 0), (19.32581, 1), (5.60514, 0), (10.20744, 1), (3.7341, 0), (6.54857, 1), (9.59001, 0), (8.69108, 1), (3.30093, 0), (8.2296, 1), (-2.75658, 0), (8.4474, 1), (4.71994, 0), (6.81178, 1), (0.74699, 0), (5.99415, 1), (2.91095, 0), (13.99336, 1), (-7.36829, 0), (8.7469, 1), (-5.29487, 0), (8.62349, 1), (3.31079, 0), (1.84212, 1), (1.06974, 0), (4.4762, 1), (-1.18424, 0), (9.25421, 1), (-7.415, 0), (10.44229, 1), (3.40595, 0), (12.21649, 1), (-7.63085, 0), (10.45968, 1), (1.13336, 0), (15.34722, 1), (-0.0096, 0), (5.50868, 1), (0.8928, 0), (10.93609, 1), (-0.5943, 0), (2.78631, 1), (7.48306, 0), (11.86145, 1), (10.11943, 0), (18.67385, 1), (5.60459, 0), (10.64051, 1), (4.00189, 0), (12.75565, 1), (2.35823, 0), (6.63666, 1), (0.33475, 0), (12.19343, 1), (3.47072, 0), (9.08636, 1), (-6.68867, 0), (11.67256, 1), (3.31031, 0), (20.31392, 1), (2.17159, 0), (11.66443, 1); SELECT '-28.740781574102936', '7.667329672103986e-133'; SELECT roundBankers(studentTTest(left, right).1, 16) as t_stat, roundBankers(studentTTest(left, right).2, 16) as p_value from student_ttest; +SELECT roundBankers(studentTTest(0.95)(left, right).3, 16) as t_stat, roundBankers(studentTTest(0.95)(left, right).4, 16) as p_value from student_ttest; DROP TABLE IF EXISTS student_ttest; diff --git a/tests/queries/0_stateless/01570_aggregator_combinator_simple_state.reference b/tests/queries/0_stateless/01570_aggregator_combinator_simple_state.reference index 1c7908bf830..351c70637c0 100644 --- a/tests/queries/0_stateless/01570_aggregator_combinator_simple_state.reference +++ b/tests/queries/0_stateless/01570_aggregator_combinator_simple_state.reference @@ -1,14 +1,31 @@ +-- { echo } +with anySimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(any, UInt64) 0 +with anyLastSimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(anyLast, UInt64) 0 +with minSimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(min, UInt64) 0 +with maxSimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(max, UInt64) 0 +with sumSimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(sum, UInt64) 0 +with sumWithOverflowSimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(sumWithOverflow, UInt64) 0 +with groupBitAndSimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(groupBitAnd, UInt64) 0 +with groupBitOrSimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(groupBitOr, UInt64) 0 +with groupBitXorSimpleState(number) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(groupBitXor, UInt64) 0 +with sumMapSimpleState(([number], [number])) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(sumMap, Tuple(Array(UInt64), Array(UInt64))) ([],[]) +with minMapSimpleState(([number], [number])) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(minMap, Tuple(Array(UInt64), Array(UInt64))) ([0],[0]) +with maxMapSimpleState(([number], [number])) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(maxMap, Tuple(Array(UInt64), Array(UInt64))) ([0],[0]) +with groupArrayArraySimpleState([number]) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(groupArrayArray, Array(UInt64)) [0] +with groupUniqArrayArraySimpleState([number]) as c select toTypeName(c), c from numbers(1); SimpleAggregateFunction(groupUniqArrayArray, Array(UInt64)) [0] +-- non-SimpleAggregateFunction +with countSimpleState(number) as c select toTypeName(c), c from numbers(1); -- { serverError 36 } diff --git a/tests/queries/0_stateless/01570_aggregator_combinator_simple_state.sql b/tests/queries/0_stateless/01570_aggregator_combinator_simple_state.sql index 00a12a69d16..94f0589670f 100644 --- a/tests/queries/0_stateless/01570_aggregator_combinator_simple_state.sql +++ b/tests/queries/0_stateless/01570_aggregator_combinator_simple_state.sql @@ -1,3 +1,4 @@ +-- { echo } with anySimpleState(number) as c select toTypeName(c), c from numbers(1); with anyLastSimpleState(number) as c select toTypeName(c), c from numbers(1); with minSimpleState(number) as c select toTypeName(c), c from numbers(1); diff --git a/tests/queries/0_stateless/01591_window_functions.reference b/tests/queries/0_stateless/01591_window_functions.reference index 4811d0a02ad..8af2c4c6b25 100644 --- a/tests/queries/0_stateless/01591_window_functions.reference +++ b/tests/queries/0_stateless/01591_window_functions.reference @@ -39,7 +39,7 @@ select number, avg(number) over (order by number rows unbounded preceding) from 8 4 9 4.5 -- no order by -select number, quantileExact(number) over (partition by intDiv(number, 3) rows unbounded preceding) from numbers(10); +select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) from numbers(10); 0 0 1 1 2 1 @@ -51,7 +51,7 @@ select number, quantileExact(number) over (partition by intDiv(number, 3) rows u 8 7 9 9 -- can add an alias after window spec -select number, quantileExact(number) over (partition by intDiv(number, 3) rows unbounded preceding) q from numbers(10); +select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) q from numbers(10); 0 0 1 1 2 1 @@ -198,7 +198,7 @@ select sum(number) over w1, sum(number) over w2 from numbers(10) window w1 as (rows unbounded preceding), - w2 as (partition by intDiv(number, 3) rows unbounded preceding) + w2 as (partition by intDiv(number, 3) as value order by number rows unbounded preceding) ; 0 0 1 1 @@ -214,7 +214,7 @@ window -- EXPLAIN test for this. select sum(number) over w1, - sum(number) over (partition by intDiv(number, 3) rows unbounded preceding) + sum(number) over (partition by intDiv(number, 3) as value order by number rows unbounded preceding) from numbers(10) window w1 as (partition by intDiv(number, 3) rows unbounded preceding) @@ -240,118 +240,118 @@ select sum(number) over () from numbers(3); -- interesting corner cases. select number, intDiv(number, 3) p, mod(number, 2) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 5 ; -0 0 0 2 +0 0 0 1 1 0 1 3 2 0 0 2 -3 1 1 3 +3 1 1 2 4 1 0 1 5 1 1 3 -6 2 0 2 +6 2 0 1 7 2 1 3 8 2 0 2 -9 3 1 3 +9 3 1 2 10 3 0 1 11 3 1 3 -12 4 0 2 +12 4 0 1 13 4 1 3 14 4 0 2 -15 5 1 3 +15 5 1 2 16 5 0 1 17 5 1 3 -18 6 0 2 +18 6 0 1 19 6 1 3 20 6 0 2 -21 7 1 3 +21 7 1 2 22 7 0 1 23 7 1 3 -24 8 0 2 +24 8 0 1 25 8 1 3 26 8 0 2 -27 9 1 3 +27 9 1 2 28 9 0 1 29 9 1 3 30 10 0 1 select number, intDiv(number, 5) p, mod(number, 3) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 2 ; -0 0 0 2 -1 0 1 4 +0 0 0 1 +1 0 1 3 2 0 2 5 3 0 0 2 4 0 1 4 -5 1 2 5 -6 1 0 2 +5 1 2 4 +6 1 0 1 7 1 1 3 8 1 2 5 9 1 0 2 -10 2 1 3 -11 2 2 5 +10 2 1 2 +11 2 2 4 12 2 0 1 13 2 1 3 14 2 2 5 -15 3 0 2 -16 3 1 4 +15 3 0 1 +16 3 1 3 17 3 2 5 18 3 0 2 19 3 1 4 -20 4 2 5 -21 4 0 2 +20 4 2 4 +21 4 0 1 22 4 1 3 23 4 2 5 24 4 0 2 -25 5 1 3 -26 5 2 5 +25 5 1 2 +26 5 2 4 27 5 0 1 28 5 1 3 29 5 2 5 30 6 0 1 select number, intDiv(number, 5) p, mod(number, 2) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 3 ; -0 0 0 3 -1 0 1 5 -2 0 0 3 +0 0 0 1 +1 0 1 4 +2 0 0 2 3 0 1 5 4 0 0 3 -5 1 1 5 -6 1 0 2 -7 1 1 5 +5 1 1 3 +6 1 0 1 +7 1 1 4 8 1 0 2 9 1 1 5 -10 2 0 3 -11 2 1 5 -12 2 0 3 +10 2 0 1 +11 2 1 4 +12 2 0 2 13 2 1 5 14 2 0 3 -15 3 1 5 -16 3 0 2 -17 3 1 5 +15 3 1 3 +16 3 0 1 +17 3 1 4 18 3 0 2 19 3 1 5 -20 4 0 3 -21 4 1 5 -22 4 0 3 +20 4 0 1 +21 4 1 4 +22 4 0 2 23 4 1 5 24 4 0 3 -25 5 1 5 -26 5 0 2 -27 5 1 5 +25 5 1 3 +26 5 0 1 +27 5 1 4 28 5 0 2 29 5 1 5 30 6 0 1 select number, intDiv(number, 3) p, mod(number, 5) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 2 ; @@ -388,7 +388,7 @@ settings max_block_size = 2 30 10 0 1 select number, intDiv(number, 2) p, mod(number, 5) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 3 ; @@ -975,39 +975,39 @@ select number, p, o, row_number() over w from (select number, intDiv(number, 5) p, mod(number, 3) o from numbers(31) order by o, number) t -window w as (partition by p order by o) +window w as (partition by p order by o, number) order by p, o, number settings max_block_size = 2; -0 0 0 2 1 1 1 -3 0 0 2 1 1 2 -1 0 1 4 3 2 3 -4 0 1 4 3 2 4 -2 0 2 5 5 3 5 -6 1 0 2 1 1 1 -9 1 0 2 1 1 2 -7 1 1 3 3 2 3 -5 1 2 5 4 3 4 -8 1 2 5 4 3 5 +0 0 0 1 1 1 1 +3 0 0 2 2 2 2 +1 0 1 3 3 3 3 +4 0 1 4 4 4 4 +2 0 2 5 5 5 5 +6 1 0 1 1 1 1 +9 1 0 2 2 2 2 +7 1 1 3 3 3 3 +5 1 2 4 4 4 4 +8 1 2 5 5 5 5 12 2 0 1 1 1 1 -10 2 1 3 2 2 2 -13 2 1 3 2 2 3 -11 2 2 5 4 3 4 -14 2 2 5 4 3 5 -15 3 0 2 1 1 2 -18 3 0 2 1 1 1 -16 3 1 4 3 2 3 -19 3 1 4 3 2 4 -17 3 2 5 5 3 5 -21 4 0 2 1 1 1 -24 4 0 2 1 1 2 -22 4 1 3 3 2 3 -20 4 2 5 4 3 5 -23 4 2 5 4 3 4 +10 2 1 2 2 2 2 +13 2 1 3 3 3 3 +11 2 2 4 4 4 4 +14 2 2 5 5 5 5 +15 3 0 1 1 1 1 +18 3 0 2 2 2 2 +16 3 1 3 3 3 3 +19 3 1 4 4 4 4 +17 3 2 5 5 5 5 +21 4 0 1 1 1 1 +24 4 0 2 2 2 2 +22 4 1 3 3 3 3 +20 4 2 4 4 4 4 +23 4 2 5 5 5 5 27 5 0 1 1 1 1 -25 5 1 3 2 2 2 -28 5 1 3 2 2 3 -26 5 2 5 4 3 4 -29 5 2 5 4 3 5 +25 5 1 2 2 2 2 +28 5 1 3 3 3 3 +26 5 2 4 4 4 4 +29 5 2 5 5 5 5 30 6 0 1 1 1 1 -- our replacement for lag/lead select @@ -1153,7 +1153,7 @@ select count() over () where null; select number, count() over (w1 rows unbounded preceding) from numbers(10) window w0 as (partition by intDiv(number, 5) as p), - w1 as (w0 order by mod(number, 3) as o) + w1 as (w0 order by mod(number, 3) as o, number) order by p, o, number ; 0 1 diff --git a/tests/queries/0_stateless/01591_window_functions.sql b/tests/queries/0_stateless/01591_window_functions.sql index aa9bd9795e7..e1e0842ad89 100644 --- a/tests/queries/0_stateless/01591_window_functions.sql +++ b/tests/queries/0_stateless/01591_window_functions.sql @@ -13,10 +13,10 @@ select number, abs(number) over (partition by toString(intDiv(number, 3)) rows u select number, avg(number) over (order by number rows unbounded preceding) from numbers(10); -- no order by -select number, quantileExact(number) over (partition by intDiv(number, 3) rows unbounded preceding) from numbers(10); +select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) from numbers(10); -- can add an alias after window spec -select number, quantileExact(number) over (partition by intDiv(number, 3) rows unbounded preceding) q from numbers(10); +select number, quantileExact(number) over (partition by intDiv(number, 3) AS value order by number rows unbounded preceding) q from numbers(10); -- can't reference it yet -- the window functions are calculated at the -- last stage of select, after all other functions. @@ -81,14 +81,14 @@ select sum(number) over w1, sum(number) over w2 from numbers(10) window w1 as (rows unbounded preceding), - w2 as (partition by intDiv(number, 3) rows unbounded preceding) + w2 as (partition by intDiv(number, 3) as value order by number rows unbounded preceding) ; -- FIXME both functions should use the same window, but they don't. Add an -- EXPLAIN test for this. select sum(number) over w1, - sum(number) over (partition by intDiv(number, 3) rows unbounded preceding) + sum(number) over (partition by intDiv(number, 3) as value order by number rows unbounded preceding) from numbers(10) window w1 as (partition by intDiv(number, 3) rows unbounded preceding) @@ -103,35 +103,35 @@ select sum(number) over () from numbers(3); -- interesting corner cases. select number, intDiv(number, 3) p, mod(number, 2) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 5 ; select number, intDiv(number, 5) p, mod(number, 3) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 2 ; select number, intDiv(number, 5) p, mod(number, 2) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 3 ; select number, intDiv(number, 3) p, mod(number, 5) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 2 ; select number, intDiv(number, 2) p, mod(number, 5) o, count(number) over w as c from numbers(31) -window w as (partition by p order by o range unbounded preceding) +window w as (partition by p order by o, number range unbounded preceding) order by number settings max_block_size = 3 ; @@ -349,7 +349,7 @@ select number, p, o, row_number() over w from (select number, intDiv(number, 5) p, mod(number, 3) o from numbers(31) order by o, number) t -window w as (partition by p order by o) +window w as (partition by p order by o, number) order by p, o, number settings max_block_size = 2; @@ -456,7 +456,7 @@ select count() over () where null; select number, count() over (w1 rows unbounded preceding) from numbers(10) window w0 as (partition by intDiv(number, 5) as p), - w1 as (w0 order by mod(number, 3) as o) + w1 as (w0 order by mod(number, 3) as o, number) order by p, o, number ; diff --git a/tests/queries/0_stateless/01592_window_functions.sql b/tests/queries/0_stateless/01592_window_functions.sql index b05b04628d2..f0d173b1f20 100644 --- a/tests/queries/0_stateless/01592_window_functions.sql +++ b/tests/queries/0_stateless/01592_window_functions.sql @@ -36,14 +36,14 @@ SELECT price, rank() OVER (PARTITION BY group_name ORDER BY price) rank FROM products INNER JOIN product_groups USING (group_id) -order by group_name, rank, price; +order by group_name, rank, price, product_name; select '---- Q3 ----'; SELECT product_name, group_name, price, - row_number() OVER (PARTITION BY group_name ORDER BY price desc) rn + row_number() OVER (PARTITION BY group_name ORDER BY price desc, product_name asc) rn FROM products INNER JOIN product_groups USING (group_id) ORDER BY group_name, rn; diff --git a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh index bc0b92eb55b..d18fbe6bdd5 100755 --- a/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh +++ b/tests/queries/0_stateless/01593_concurrent_alter_mutations_kill.sh @@ -67,5 +67,4 @@ done $CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE concurrent_mutate_kill" $CLICKHOUSE_CLIENT --query "OPTIMIZE TABLE concurrent_mutate_kill FINAL" $CLICKHOUSE_CLIENT --query "SELECT sum(value) FROM concurrent_mutate_kill" - $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS concurrent_mutate_kill" diff --git a/tests/queries/0_stateless/01670_neighbor_lc_bug.sql b/tests/queries/0_stateless/01670_neighbor_lc_bug.sql index 2d99225aa89..f216befbb06 100644 --- a/tests/queries/0_stateless/01670_neighbor_lc_bug.sql +++ b/tests/queries/0_stateless/01670_neighbor_lc_bug.sql @@ -37,7 +37,9 @@ FROM ( SELECT * FROM neighbor_test - ORDER BY val_string ASC -) format PrettyCompact; + ORDER BY val_string, rowNr +) +ORDER BY rowNr, val_string, str_m1, str_p1, val_low, low_m1, low_p1 +format PrettyCompact; drop table if exists neighbor_test; diff --git a/tests/queries/0_stateless/01671_merge_join_and_constants.sql b/tests/queries/0_stateless/01671_merge_join_and_constants.sql index c34f8e6705d..e9a60f11875 100644 --- a/tests/queries/0_stateless/01671_merge_join_and_constants.sql +++ b/tests/queries/0_stateless/01671_merge_join_and_constants.sql @@ -9,7 +9,7 @@ INSERT INTO table2 VALUES ('D', 'd', '2018-01-01') ('B', 'b', '2018-01-01') ('C' set join_algorithm = 'partial_merge'; -SELECT * FROM table1 AS t1 ALL LEFT JOIN (SELECT *, '0.10', c, d AS b FROM table2) AS t2 USING (a, b) ORDER BY d ASC FORMAT PrettyCompact settings max_rows_in_join = 1; +SELECT * FROM table1 AS t1 ALL LEFT JOIN (SELECT *, '0.10', c, d AS b FROM table2) AS t2 USING (a, b) ORDER BY d, t1.a ASC FORMAT PrettyCompact settings max_rows_in_join = 1; SELECT pow('0.0000000257', NULL), pow(pow(NULL, NULL), NULL) - NULL, (val + NULL) = (rval * 0), * FROM (SELECT (val + 256) = (NULL * NULL), toLowCardinality(toNullable(dummy)) AS val FROM system.one) AS s1 ANY LEFT JOIN (SELECT toLowCardinality(dummy) AS rval FROM system.one) AS s2 ON (val + 0) = (rval * 255) settings max_rows_in_join = 1; diff --git a/tests/queries/0_stateless/01700_deltasum.sql b/tests/queries/0_stateless/01700_deltasum.sql index 83d5e0439d2..9f1404c6845 100644 --- a/tests/queries/0_stateless/01700_deltasum.sql +++ b/tests/queries/0_stateless/01700_deltasum.sql @@ -3,8 +3,42 @@ select deltaSum(arrayJoin([1, 2, 3, 0, 3, 4])); select deltaSum(arrayJoin([1, 2, 3, 0, 3, 4, 2, 3])); select deltaSum(arrayJoin([1, 2, 3, 0, 3, 3, 3, 3, 3, 4, 2, 3])); select deltaSum(arrayJoin([1, 2, 3, 0, 0, 0, 0, 3, 3, 3, 3, 3, 4, 2, 3])); -select deltaSumMerge(rows) from (select deltaSumState(arrayJoin([0, 1])) as rows union all select deltaSumState(arrayJoin([4, 5])) as rows); -select deltaSumMerge(rows) from (select deltaSumState(arrayJoin([4, 5])) as rows union all select deltaSumState(arrayJoin([0, 1])) as rows); +select deltaSumMerge(rows) from +( + select * from + ( + select deltaSumState(arrayJoin([0, 1])) as rows + union all + select deltaSumState(arrayJoin([4, 5])) as rows + ) order by rows +); +select deltaSumMerge(rows) from +( + select * from + ( + select deltaSumState(arrayJoin([4, 5])) as rows + union all + select deltaSumState(arrayJoin([0, 1])) as rows + ) order by rows +); select deltaSum(arrayJoin([2.25, 3, 4.5])); -select deltaSumMerge(rows) from (select deltaSumState(arrayJoin([0.1, 0.3, 0.5])) as rows union all select deltaSumState(arrayJoin([4.1, 5.1, 6.6])) as rows); -select deltaSumMerge(rows) from (select deltaSumState(arrayJoin([3, 5])) as rows union all select deltaSumState(arrayJoin([1, 2])) as rows union all select deltaSumState(arrayJoin([4, 6])) as rows); +select deltaSumMerge(rows) from +( + select * from + ( + select deltaSumState(arrayJoin([0.1, 0.3, 0.5])) as rows + union all + select deltaSumState(arrayJoin([4.1, 5.1, 6.6])) as rows + ) order by rows +); +select deltaSumMerge(rows) from +( + select * from + ( + select deltaSumState(arrayJoin([3, 5])) as rows + union all + select deltaSumState(arrayJoin([1, 2])) as rows + union all + select deltaSumState(arrayJoin([4, 6])) as rows + ) order by rows +); diff --git a/tests/queries/0_stateless/01702_system_query_log.reference b/tests/queries/0_stateless/01702_system_query_log.reference index c9d75db01c0..1f329feac22 100644 --- a/tests/queries/0_stateless/01702_system_query_log.reference +++ b/tests/queries/0_stateless/01702_system_query_log.reference @@ -42,7 +42,7 @@ Alter ALTER TABLE sqllt.table DROP COLUMN the_new_col; Alter ALTER TABLE sqllt.table UPDATE i = i + 1 WHERE 1; Alter ALTER TABLE sqllt.table DELETE WHERE i > 65535; Select -- not done, seems to hard, so I\'ve skipped queries of ALTER-X, where X is:\n-- PARTITION\n-- ORDER BY\n-- SAMPLE BY\n-- INDEX\n-- CONSTRAINT\n-- TTL\n-- USER\n-- QUOTA\n-- ROLE\n-- ROW POLICY\n-- SETTINGS PROFILE\n\nSELECT \'SYSTEM queries\'; -System SYSTEM FLUSH LOGS +System SYSTEM FLUSH LOGS; System SYSTEM STOP MERGES sqllt.table System SYSTEM START MERGES sqllt.table System SYSTEM STOP TTL MERGES sqllt.table diff --git a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference index 00c77f1500c..b6f5fe99ca1 100644 --- a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference +++ b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.reference @@ -1,2 +1,2 @@ CREATE TABLE default.x\n(\n `i` Int32,\n INDEX mm rand() TYPE minmax GRANULARITY 1,\n INDEX nn rand() TYPE minmax GRANULARITY 1,\n PROJECTION p\n (\n SELECT max(i)\n ),\n PROJECTION p2\n (\n SELECT min(i)\n )\n)\nENGINE = ReplicatedMergeTree(\'/clickhouse/tables/default/x\', \'r\')\nORDER BY i\nSETTINGS index_granularity = 8192 -metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 8192\nmode: 0\nsign column: \nprimary key: i\ndata format version: 1\npartition key: \nindices: mm rand() TYPE minmax GRANULARITY 1, nn rand() TYPE minmax GRANULARITY 1\nprojections: p(SELECT max(i)), p2(SELECT min(i))\ngranularity bytes: 10485760\n +metadata format version: 1\ndate column: \nsampling expression: \nindex granularity: 8192\nmode: 0\nsign column: \nprimary key: i\ndata format version: 1\npartition key: \nindices: mm rand() TYPE minmax GRANULARITY 1, nn rand() TYPE minmax GRANULARITY 1\nprojections: p (SELECT max(i)), p2 (SELECT min(i))\ngranularity bytes: 10485760\n diff --git a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql index 59993f40774..683bd271405 100644 --- a/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql +++ b/tests/queries/0_stateless/01705_normalize_create_alter_function_names.sql @@ -1,4 +1,4 @@ --- Tags: zookeeper, no-replicated-database, no-parallel +-- Tags: zookeeper, no-replicated-database, no-parallel, no-s3-storage drop table if exists x; diff --git a/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql b/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql index d5eaa2617a6..fd17bdf1e3c 100644 --- a/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql +++ b/tests/queries/0_stateless/01710_aggregate_projection_with_hashing.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage set allow_experimental_projection_optimization = 1, force_optimize_projection = 1; drop table if exists tp; diff --git a/tests/queries/0_stateless/01710_aggregate_projections.sh b/tests/queries/0_stateless/01710_aggregate_projections.sh index a8b3e6bf99d..f4b2c2189dd 100755 --- a/tests/queries/0_stateless/01710_aggregate_projections.sh +++ b/tests/queries/0_stateless/01710_aggregate_projections.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# Tags: no-s3-storage CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -14,8 +15,8 @@ $CLICKHOUSE_CLIENT -q "select x + y, sum(x - y) as s from test_agg_proj group by $CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1" $CLICKHOUSE_CLIENT -q "select (x + y) * 2, sum(x - y) * 2 as s from test_agg_proj group by x + y order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read" -$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2), intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc limit 5 settings allow_experimental_projection_optimization=1" -$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2), intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read" +$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings allow_experimental_projection_optimization=1" +$CLICKHOUSE_CLIENT -q "select intDiv(x + y, 2) as v, intDiv(x + y, 3), sum(x - y) as s from test_agg_proj group by intDiv(x + y, 2), intDiv(x + y, 3) order by s desc, v limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read" $CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1" $CLICKHOUSE_CLIENT -q "select x + y + 1, argMax(x, y) * sum(x - y) as s from test_agg_proj group by x + y + 1 order by s desc limit 5 settings allow_experimental_projection_optimization=1 format JSON" | grep "rows_read" diff --git a/tests/queries/0_stateless/01710_force_use_projection.sql b/tests/queries/0_stateless/01710_force_use_projection.sql index 8931c65e34e..90dd4a6bec3 100644 --- a/tests/queries/0_stateless/01710_force_use_projection.sql +++ b/tests/queries/0_stateless/01710_force_use_projection.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists tp; create table tp (d1 Int32, d2 Int32, eventcnt Int64, projection p (select sum(eventcnt) group by d1)) engine = MergeTree order by (d1, d2); diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.reference b/tests/queries/0_stateless/01710_minmax_count_projection.reference index 77649f536f5..b13738a66de 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.reference +++ b/tests/queries/0_stateless/01710_minmax_count_projection.reference @@ -13,7 +13,7 @@ 1 1 1 -\N 2021-10-27 10:00:00 4 -2021-10-24 10:00:00 +\N 2021-10-27 10:00:00 3 +0 2021-10-24 10:00:00 0 diff --git a/tests/queries/0_stateless/01710_minmax_count_projection.sql b/tests/queries/0_stateless/01710_minmax_count_projection.sql index 713241ada72..0792fe331bb 100644 --- a/tests/queries/0_stateless/01710_minmax_count_projection.sql +++ b/tests/queries/0_stateless/01710_minmax_count_projection.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists d; create table d (i int, j int) engine MergeTree partition by i % 2 order by tuple() settings index_granularity = 1; @@ -53,7 +54,7 @@ select count() from d group by toDate(dt); -- fuzz crash SELECT pointInEllipses(min(j), NULL), max(dt), count('0.0000000007') FROM d WHERE toDate(dt) >= '2021-10-25'; -SELECT min(dt) FROM d PREWHERE ceil(j) <= 0; +SELECT min(j) FROM d PREWHERE ceil(j) <= 0; SELECT min(dt) FROM d PREWHERE ((0.9998999834060669 AND 1023) AND 255) <= ceil(j); SELECT count('') AND NULL FROM d PREWHERE ceil(j) <= NULL; diff --git a/tests/queries/0_stateless/01710_normal_projection_fix1.sql b/tests/queries/0_stateless/01710_normal_projection_fix1.sql index b4d7c6e8734..0634a810619 100644 --- a/tests/queries/0_stateless/01710_normal_projection_fix1.sql +++ b/tests/queries/0_stateless/01710_normal_projection_fix1.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists t; create table t (i int, j int) engine MergeTree order by i; diff --git a/tests/queries/0_stateless/01710_normal_projections.sh b/tests/queries/0_stateless/01710_normal_projections.sh index 70e38b3722a..1ab659a76ab 100755 --- a/tests/queries/0_stateless/01710_normal_projections.sh +++ b/tests/queries/0_stateless/01710_normal_projections.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# Tags: no-s3-storage + CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh diff --git a/tests/queries/0_stateless/01710_projection_aggregation_in_order.reference b/tests/queries/0_stateless/01710_projection_aggregation_in_order.reference new file mode 100644 index 00000000000..12c613c184d --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_aggregation_in_order.reference @@ -0,0 +1,20 @@ +276078600 +291519000 +304558200 +317518200 +330478200 +276078600 +291519000 +304558200 +317518200 +330478200 +276078600 +291519000 +304558200 +317518200 +330478200 +276078600 +291519000 +304558200 +317518200 +330478200 diff --git a/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql new file mode 100644 index 00000000000..557bd297436 --- /dev/null +++ b/tests/queries/0_stateless/01710_projection_aggregation_in_order.sql @@ -0,0 +1,59 @@ +DROP TABLE IF EXISTS normal; + +CREATE TABLE normal +( + `key` UInt32, + `ts` DateTime, + `value` UInt32, + PROJECTION aaaa + ( + SELECT + ts, + key, + value + ORDER BY ts, key + ) +) +ENGINE = MergeTree +ORDER BY (key, ts); + +INSERT INTO normal SELECT + 1, + toDateTime('2021-12-06 00:00:00') + number, + number +FROM numbers(100000); + +SET allow_experimental_projection_optimization=1, optimize_aggregation_in_order=1, force_optimize_projection=1; + +WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM normal WHERE ts > '2021-12-06 22:00:00' GROUP BY a ORDER BY v LIMIT 5; +WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM normal WHERE ts > '2021-12-06 22:00:00' GROUP BY toStartOfHour(ts), a ORDER BY v LIMIT 5; + +DROP TABLE IF EXISTS agg; + +CREATE TABLE agg +( + `key` UInt32, + `ts` DateTime, + `value` UInt32, + PROJECTION aaaa + ( + SELECT + ts, + key, + sum(value) + GROUP BY ts, key + ) +) +ENGINE = MergeTree +ORDER BY (key, ts); + +INSERT INTO agg SELECT + 1, + toDateTime('2021-12-06 00:00:00') + number, + number +FROM numbers(100000); + +SET allow_experimental_projection_optimization=1, optimize_aggregation_in_order=1, force_optimize_projection = 1; + +WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM normal WHERE ts > '2021-12-06 22:00:00' GROUP BY a ORDER BY v LIMIT 5; +WITH toStartOfHour(ts) AS a SELECT sum(value) v FROM normal WHERE ts > '2021-12-06 22:00:00' GROUP BY toStartOfHour(ts), a ORDER BY v LIMIT 5; diff --git a/tests/queries/0_stateless/01710_projection_array_join.sql b/tests/queries/0_stateless/01710_projection_array_join.sql index cd18d9282b9..fde355fe4eb 100644 --- a/tests/queries/0_stateless/01710_projection_array_join.sql +++ b/tests/queries/0_stateless/01710_projection_array_join.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage set allow_experimental_projection_optimization = 1; drop table if exists x; diff --git a/tests/queries/0_stateless/01710_projection_detach_part.sql b/tests/queries/0_stateless/01710_projection_detach_part.sql index e3e6c7ac165..73e801a11ea 100644 --- a/tests/queries/0_stateless/01710_projection_detach_part.sql +++ b/tests/queries/0_stateless/01710_projection_detach_part.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage set allow_experimental_projection_optimization = 1; drop table if exists t; diff --git a/tests/queries/0_stateless/01710_projection_drop_if_exists.sql b/tests/queries/0_stateless/01710_projection_drop_if_exists.sql index f21092e5491..93b121dd39c 100644 --- a/tests/queries/0_stateless/01710_projection_drop_if_exists.sql +++ b/tests/queries/0_stateless/01710_projection_drop_if_exists.sql @@ -1,3 +1,5 @@ +-- Tags: no-s3-storage + drop table if exists tp; create table tp (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y; diff --git a/tests/queries/0_stateless/01710_projection_fetch_long.sql b/tests/queries/0_stateless/01710_projection_fetch_long.sql index 6c41c69254e..fd12b84c817 100644 --- a/tests/queries/0_stateless/01710_projection_fetch_long.sql +++ b/tests/queries/0_stateless/01710_projection_fetch_long.sql @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-s3-storage drop table if exists tp_1; drop table if exists tp_2; diff --git a/tests/queries/0_stateless/01710_projection_group_by_order_by.sql b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql index 9370e9d36ce..d45b959ccd5 100644 --- a/tests/queries/0_stateless/01710_projection_group_by_order_by.sql +++ b/tests/queries/0_stateless/01710_projection_group_by_order_by.sql @@ -1,3 +1,6 @@ +-- Tags: no-s3-storage + +DROP TABLE IF EXISTS t; drop table if exists tp; create table tp (type Int32, eventcnt UInt64, projection p (select sum(eventcnt), type group by type order by sum(eventcnt))) engine = MergeTree order by type; -- { serverError 583 } diff --git a/tests/queries/0_stateless/01710_projection_in_index.sql b/tests/queries/0_stateless/01710_projection_in_index.sql index 2669d69dc9f..e1ae3540705 100644 --- a/tests/queries/0_stateless/01710_projection_in_index.sql +++ b/tests/queries/0_stateless/01710_projection_in_index.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists t; create table t (i int, j int, k int, projection p (select * order by j)) engine MergeTree order by i settings index_granularity = 1; diff --git a/tests/queries/0_stateless/01710_projection_in_set.sql b/tests/queries/0_stateless/01710_projection_in_set.sql index 99fa2cab0c5..39b54db86e3 100644 --- a/tests/queries/0_stateless/01710_projection_in_set.sql +++ b/tests/queries/0_stateless/01710_projection_in_set.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists x; create table x (i UInt64, j UInt64, k UInt64, projection agg (select sum(j), avg(k) group by i), projection norm (select j, k order by i)) engine MergeTree order by tuple(); diff --git a/tests/queries/0_stateless/01710_projection_materialize_with_missing_columns.sql b/tests/queries/0_stateless/01710_projection_materialize_with_missing_columns.sql index 28bf1c050d0..4b2675f9677 100644 --- a/tests/queries/0_stateless/01710_projection_materialize_with_missing_columns.sql +++ b/tests/queries/0_stateless/01710_projection_materialize_with_missing_columns.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists x; create table x (i int) engine MergeTree order by tuple(); diff --git a/tests/queries/0_stateless/01710_projection_mutation.sql b/tests/queries/0_stateless/01710_projection_mutation.sql index ab3fbd117d0..2503d828880 100644 --- a/tests/queries/0_stateless/01710_projection_mutation.sql +++ b/tests/queries/0_stateless/01710_projection_mutation.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS t; CREATE TABLE t (`key` UInt32, `created_at` Date, `value` UInt32, PROJECTION xxx (SELECT key, created_at, sum(value) GROUP BY key, created_at)) ENGINE = MergeTree PARTITION BY toYYYYMM(created_at) ORDER BY key; @@ -5,3 +6,5 @@ CREATE TABLE t (`key` UInt32, `created_at` Date, `value` UInt32, PROJECTION xxx INSERT INTO t SELECT 1 AS key, today() + (number % 30), number FROM numbers(1000); ALTER TABLE t UPDATE value = 0 WHERE (value > 0) AND (created_at >= '2021-12-21') SETTINGS allow_experimental_projection_optimization = 1; + +DROP TABLE IF EXISTS t; diff --git a/tests/queries/0_stateless/01710_projection_optimize_materialize.sql b/tests/queries/0_stateless/01710_projection_optimize_materialize.sql index d8251aabaf6..27252e2a171 100644 --- a/tests/queries/0_stateless/01710_projection_optimize_materialize.sql +++ b/tests/queries/0_stateless/01710_projection_optimize_materialize.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists z; create table z (pk Int64, d Date, id UInt64, c UInt64) Engine MergeTree partition by d order by pk ; diff --git a/tests/queries/0_stateless/01710_projection_part_check.sql b/tests/queries/0_stateless/01710_projection_part_check.sql index 1ccd9de5903..aa087169ad1 100644 --- a/tests/queries/0_stateless/01710_projection_part_check.sql +++ b/tests/queries/0_stateless/01710_projection_part_check.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists tp; create table tp (x Int32, y Int32, projection p (select x, y order by x)) engine = MergeTree order by y settings min_rows_for_compact_part = 2, min_rows_for_wide_part = 4, min_bytes_for_compact_part = 16, min_bytes_for_wide_part = 32; diff --git a/tests/queries/0_stateless/01710_projection_vertical_merges.sql b/tests/queries/0_stateless/01710_projection_vertical_merges.sql index d54fef7e71d..74e3e3ac6ce 100644 --- a/tests/queries/0_stateless/01710_projection_vertical_merges.sql +++ b/tests/queries/0_stateless/01710_projection_vertical_merges.sql @@ -1,4 +1,4 @@ --- Tags: long, no-parallel +-- Tags: long, no-parallel, no-s3-storage drop table if exists t; diff --git a/tests/queries/0_stateless/01710_projection_with_joins.reference b/tests/queries/0_stateless/01710_projection_with_joins.reference index e69de29bb2d..4792e70f333 100644 --- a/tests/queries/0_stateless/01710_projection_with_joins.reference +++ b/tests/queries/0_stateless/01710_projection_with_joins.reference @@ -0,0 +1,2 @@ +2 +3 diff --git a/tests/queries/0_stateless/01710_projection_with_joins.sql b/tests/queries/0_stateless/01710_projection_with_joins.sql index fcd1c586fa3..472242e3043 100644 --- a/tests/queries/0_stateless/01710_projection_with_joins.sql +++ b/tests/queries/0_stateless/01710_projection_with_joins.sql @@ -1,8 +1,22 @@ +-- Tags: no-s3-storage drop table if exists t; -create table t (s UInt16, l UInt16, projection p (select s, l order by l)) engine MergeTree order by s; +create table t (s UInt16, l UInt16, projection p (select s, l order by l)) engine MergeTree order by s; select s from t join (select toUInt16(1) as s) x using (s) settings allow_experimental_projection_optimization = 1; select s from t join (select toUInt16(1) as s) x using (s) settings allow_experimental_projection_optimization = 0; drop table t; + +drop table if exists mt; +create table mt (id1 Int8, id2 Int8) Engine=MergeTree order by tuple(); +select id1 as alias1 from mt all inner join (select id2 as alias1 from mt) as t using (alias1) settings allow_experimental_projection_optimization = 1; +select id1 from mt all inner join (select id2 as id1 from mt) as t using (id1) settings allow_experimental_projection_optimization = 1; +select id2 as id1 from mt all inner join (select id1 from mt) as t using (id1) settings allow_experimental_projection_optimization = 1; +drop table mt; + +drop table if exists j; +create table j (id1 Int8, id2 Int8, projection p (select id1, id2 order by id2)) Engine=MergeTree order by id1 settings index_granularity = 1; +insert into j select number, number from numbers(10); +select id1 as alias1 from j all inner join (select id2 as alias1 from j where id2 in (1, 2, 3)) as t using (alias1) where id2 in (2, 3, 4) settings allow_experimental_projection_optimization = 1; +drop table j; diff --git a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql index 734aa659146..7986ca40095 100644 --- a/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql +++ b/tests/queries/0_stateless/01710_projection_with_mixed_pipeline.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists t; create table t (x UInt32) engine = MergeTree order by tuple() settings index_granularity = 8; diff --git a/tests/queries/0_stateless/01710_projections.sql b/tests/queries/0_stateless/01710_projections.sql index c929bfaa273..54581b5ae11 100644 --- a/tests/queries/0_stateless/01710_projections.sql +++ b/tests/queries/0_stateless/01710_projections.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists projection_test; create table projection_test (`sum(block_count)` UInt64, domain_alias UInt64 alias length(domain), datetime DateTime, domain LowCardinality(String), x_id String, y_id String, block_count Int64, retry_count Int64, duration Int64, kbytes Int64, buffer_time Int64, first_time Int64, total_bytes Nullable(UInt64), valid_bytes Nullable(UInt64), completed_bytes Nullable(UInt64), fixed_bytes Nullable(UInt64), force_bytes Nullable(UInt64), projection p (select toStartOfMinute(datetime) dt_m, countIf(first_time = 0) / count(), avg((kbytes * 8) / duration), count(), sum(block_count) / sum(duration), avg(block_count / duration), sum(buffer_time) / sum(duration), avg(buffer_time / duration), sum(valid_bytes) / sum(total_bytes), sum(completed_bytes) / sum(total_bytes), sum(fixed_bytes) / sum(total_bytes), sum(force_bytes) / sum(total_bytes), sum(valid_bytes) / sum(total_bytes), sum(retry_count) / sum(duration), avg(retry_count / duration), countIf(block_count > 0) / count(), countIf(first_time = 0) / count(), uniqHLL12(x_id), uniqHLL12(y_id) group by dt_m, domain)) engine MergeTree partition by toDate(datetime) order by (toStartOfTenMinutes(datetime), domain); diff --git a/tests/queries/0_stateless/01710_projections_group_by_no_key.sql b/tests/queries/0_stateless/01710_projections_group_by_no_key.sql index eefc03afb7a..b91d182e309 100644 --- a/tests/queries/0_stateless/01710_projections_group_by_no_key.sql +++ b/tests/queries/0_stateless/01710_projections_group_by_no_key.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage drop table if exists projection_without_key; create table projection_without_key (key UInt32, PROJECTION x (SELECT sum(key) group by key % 3)) engine MergeTree order by key; diff --git a/tests/queries/0_stateless/01710_projections_in_distributed_query.sql b/tests/queries/0_stateless/01710_projections_in_distributed_query.sql index fa734b605cd..71ac158577a 100644 --- a/tests/queries/0_stateless/01710_projections_in_distributed_query.sql +++ b/tests/queries/0_stateless/01710_projections_in_distributed_query.sql @@ -1,4 +1,4 @@ --- Tags: distributed +-- Tags: distributed, no-s3-storage drop table if exists projection_test; diff --git a/tests/queries/0_stateless/01739_index_hint.reference b/tests/queries/0_stateless/01739_index_hint.reference index 6aa40c5d302..71dfab29154 100644 --- a/tests/queries/0_stateless/01739_index_hint.reference +++ b/tests/queries/0_stateless/01739_index_hint.reference @@ -25,8 +25,8 @@ drop table tbl; drop table if exists XXXX; create table XXXX (t Int64, f Float64) Engine=MergeTree order by t settings index_granularity=128; insert into XXXX select number*60, 0 from numbers(100000); -SELECT count() FROM XXXX WHERE indexHint(t = 42); -128 +SELECT sum(t) FROM XXXX WHERE indexHint(t = 42); +487680 drop table if exists XXXX; create table XXXX (t Int64, f Float64) Engine=MergeTree order by t settings index_granularity=8192; insert into XXXX select number*60, 0 from numbers(100000); diff --git a/tests/queries/0_stateless/01739_index_hint.sql b/tests/queries/0_stateless/01739_index_hint.sql index 28395c2dc1d..30dfa43d334 100644 --- a/tests/queries/0_stateless/01739_index_hint.sql +++ b/tests/queries/0_stateless/01739_index_hint.sql @@ -22,7 +22,7 @@ create table XXXX (t Int64, f Float64) Engine=MergeTree order by t settings inde insert into XXXX select number*60, 0 from numbers(100000); -SELECT count() FROM XXXX WHERE indexHint(t = 42); +SELECT sum(t) FROM XXXX WHERE indexHint(t = 42); drop table if exists XXXX; diff --git a/tests/queries/0_stateless/01748_partition_id_pruning.sql b/tests/queries/0_stateless/01748_partition_id_pruning.sql index e0d45884c60..9a26dd8daba 100644 --- a/tests/queries/0_stateless/01748_partition_id_pruning.sql +++ b/tests/queries/0_stateless/01748_partition_id_pruning.sql @@ -14,6 +14,8 @@ select * from x where _partition_id in (select partitionId(number + 1) from numb -- trivial count optimization test set max_rows_to_read = 2; -- one row for subquery + subquery itself +-- TODO: Relax the limits because we might build prepared set twice with _minmax_count_projection +set max_rows_to_read = 3; select count() from x where _partition_id in (select partitionId(number + 1) from numbers(1)); drop table x; diff --git a/tests/queries/0_stateless/01780_column_sparse.reference b/tests/queries/0_stateless/01780_column_sparse.reference index 08aef433172..1cbcf715d7f 100644 --- a/tests/queries/0_stateless/01780_column_sparse.reference +++ b/tests/queries/0_stateless/01780_column_sparse.reference @@ -142,7 +142,7 @@ CREATE TABLE t_sparse_1 (id UInt64, v Int64) ENGINE = MergeTree ORDER BY tuple() SETTINGS ratio_of_defaults_for_sparse_serialization = 0; INSERT INTO t_sparse_1 VALUES (1, 6), (2, 1), (3, 0), (4, -1), (5, 0), (6, 0), (7, -2), (8, 0), (9, 0), (10, 4), (11, 0); -SELECT * FROM t_sparse_1 ORDER BY v; +SELECT * FROM t_sparse_1 ORDER BY v, id; 7 -2 4 -1 3 0 @@ -154,7 +154,7 @@ SELECT * FROM t_sparse_1 ORDER BY v; 2 1 10 4 1 6 -SELECT * FROM t_sparse_1 ORDER BY v DESC; +SELECT * FROM t_sparse_1 ORDER BY v DESC, id; 1 6 10 4 2 1 diff --git a/tests/queries/0_stateless/01780_column_sparse.sql b/tests/queries/0_stateless/01780_column_sparse.sql index 480321c6d14..25cb2845322 100644 --- a/tests/queries/0_stateless/01780_column_sparse.sql +++ b/tests/queries/0_stateless/01780_column_sparse.sql @@ -35,8 +35,8 @@ SETTINGS ratio_of_defaults_for_sparse_serialization = 0; INSERT INTO t_sparse_1 VALUES (1, 6), (2, 1), (3, 0), (4, -1), (5, 0), (6, 0), (7, -2), (8, 0), (9, 0), (10, 4), (11, 0); -SELECT * FROM t_sparse_1 ORDER BY v; -SELECT * FROM t_sparse_1 ORDER BY v DESC; +SELECT * FROM t_sparse_1 ORDER BY v, id; +SELECT * FROM t_sparse_1 ORDER BY v DESC, id; SELECT * FROM t_sparse_1 ORDER BY v, id LIMIT 5; SELECT * FROM t_sparse_1 ORDER BY v DESC, id LIMIT 5; diff --git a/tests/queries/0_stateless/01780_column_sparse_alter.reference b/tests/queries/0_stateless/01780_column_sparse_alter.reference index cec7af647b3..4fb0122db96 100644 --- a/tests/queries/0_stateless/01780_column_sparse_alter.reference +++ b/tests/queries/0_stateless/01780_column_sparse_alter.reference @@ -1,6 +1,6 @@ id Default -u Sparse s Sparse +u Sparse 182 155 id Default t Sparse diff --git a/tests/queries/0_stateless/01780_column_sparse_alter.sql b/tests/queries/0_stateless/01780_column_sparse_alter.sql index 444a1f9cf43..7f9558bfc18 100644 --- a/tests/queries/0_stateless/01780_column_sparse_alter.sql +++ b/tests/queries/0_stateless/01780_column_sparse_alter.sql @@ -12,14 +12,14 @@ INSERT INTO t_sparse_alter SELECT if (number % 13 = 0, toString(number), '') FROM numbers(2000); -SELECT column, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_sparse_alter' AND active ORDER BY name; +SELECT column, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_sparse_alter' AND active ORDER BY column; SELECT uniqExact(u), uniqExact(s) FROM t_sparse_alter; ALTER TABLE t_sparse_alter DROP COLUMN s, RENAME COLUMN u TO t; ALTER TABLE t_sparse_alter MODIFY COLUMN t UInt16; -SELECT column, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_sparse_alter' AND active ORDER BY name; +SELECT column, serialization_kind FROM system.parts_columns WHERE database = currentDatabase() AND table = 't_sparse_alter' AND active ORDER BY column; SELECT uniqExact(t) FROM t_sparse_alter; diff --git a/tests/queries/0_stateless/01780_column_sparse_filter.reference b/tests/queries/0_stateless/01780_column_sparse_filter.reference new file mode 100644 index 00000000000..d673acfc89f --- /dev/null +++ b/tests/queries/0_stateless/01780_column_sparse_filter.reference @@ -0,0 +1,13 @@ +id Default +s Sparse +u Sparse +5000 +2000 +id Default +id Default +s Default +s Sparse +u Default +u Sparse +105000 +102000 diff --git a/tests/queries/0_stateless/01780_column_sparse_filter.sql b/tests/queries/0_stateless/01780_column_sparse_filter.sql new file mode 100644 index 00000000000..45958b5c4e0 --- /dev/null +++ b/tests/queries/0_stateless/01780_column_sparse_filter.sql @@ -0,0 +1,33 @@ +DROP TABLE IF EXISTS t_sparse; + +CREATE TABLE t_sparse (id UInt64, u UInt64, s String) +ENGINE = MergeTree ORDER BY id +SETTINGS ratio_of_defaults_for_sparse_serialization = 0.9; + +INSERT INTO t_sparse SELECT + number, + if (number % 20 = 0, number, 0), + if (number % 50 = 0, toString(number), '') +FROM numbers(1, 100000); + +SELECT column, serialization_kind FROM system.parts_columns +WHERE table = 't_sparse' AND database = currentDatabase() +ORDER BY column, serialization_kind; + +SELECT count() FROM t_sparse WHERE u > 0; +SELECT count() FROM t_sparse WHERE notEmpty(s); + +SYSTEM STOP MERGES t_sparse; + +INSERT INTO t_sparse SELECT + number, number, toString(number) +FROM numbers (1, 100000); + +SELECT column, serialization_kind FROM system.parts_columns +WHERE table = 't_sparse' AND database = currentDatabase() +ORDER BY column, serialization_kind; + +SELECT count() FROM t_sparse WHERE u > 0; +SELECT count() FROM t_sparse WHERE notEmpty(s); + +DROP TABLE t_sparse; diff --git a/tests/queries/0_stateless/01780_column_sparse_full.reference b/tests/queries/0_stateless/01780_column_sparse_full.reference index 4d2d0a58798..dbe815800a9 100644 --- a/tests/queries/0_stateless/01780_column_sparse_full.reference +++ b/tests/queries/0_stateless/01780_column_sparse_full.reference @@ -91,9 +91,9 @@ all_2_2_0 u Default ====== 0 0 0 0 0 0 -1 1 1 1 0 -2 2 2 +1 1 1 +2 0 ====== 0 0 0 0 0 0 diff --git a/tests/queries/0_stateless/01780_column_sparse_full.sql b/tests/queries/0_stateless/01780_column_sparse_full.sql index af6fde116d9..59128223dba 100644 --- a/tests/queries/0_stateless/01780_column_sparse_full.sql +++ b/tests/queries/0_stateless/01780_column_sparse_full.sql @@ -81,7 +81,7 @@ INNER JOIN t_sparse_full USING(u) ORDER BY id, u, s LIMIT 5; SELECT '======'; SELECT id, u, s FROM (SELECT number * 2 AS u FROM numbers(10)) AS t1 -FULL JOIN t_sparse_full USING(u) ORDER BY id LIMIT 5; +FULL JOIN t_sparse_full USING(u) ORDER BY id, u, s LIMIT 5; SELECT '======'; diff --git a/tests/queries/0_stateless/01780_column_sparse_tuple.sql b/tests/queries/0_stateless/01780_column_sparse_tuple.sql index da679f2c7eb..e3dfc16fc74 100644 --- a/tests/queries/0_stateless/01780_column_sparse_tuple.sql +++ b/tests/queries/0_stateless/01780_column_sparse_tuple.sql @@ -1,3 +1,4 @@ +-- Tags: no-s3-storage DROP TABLE IF EXISTS sparse_tuple; CREATE TABLE sparse_tuple (id UInt64, t Tuple(a UInt64, s String)) diff --git a/tests/queries/0_stateless/01810_max_part_removal_threads_long.sh b/tests/queries/0_stateless/01810_max_part_removal_threads_long.sh index c5aaa794ac9..f5ab71d8d34 100755 --- a/tests/queries/0_stateless/01810_max_part_removal_threads_long.sh +++ b/tests/queries/0_stateless/01810_max_part_removal_threads_long.sh @@ -16,22 +16,54 @@ $CLICKHOUSE_CLIENT -nm -q "create database ordinary_$CLICKHOUSE_DATABASE engine= $CLICKHOUSE_CLIENT -nm -q """ use ordinary_$CLICKHOUSE_DATABASE; drop table if exists data_01810; - create table data_01810 (key Int) Engine=MergeTree() order by key partition by key settings max_part_removal_threads=10, concurrent_part_removal_threshold=49; - insert into data_01810 select * from numbers(50); + + create table data_01810 (key Int) + Engine=MergeTree() + order by key + partition by key%100 + settings max_part_removal_threads=10, concurrent_part_removal_threshold=99, min_bytes_for_wide_part=0; + + insert into data_01810 select * from numbers(100); drop table data_01810 settings log_queries=1; system flush logs; - select throwIf(length(thread_ids)<50) from system.query_log where event_date >= yesterday() and current_database = currentDatabase() and query = 'drop table data_01810 settings log_queries=1;' and type = 'QueryFinish' format Null; + + -- sometimes the same thread can be used to remove part, due to ThreadPool, + -- hence we cannot compare strictly. + select throwIf(not(length(thread_ids) between 6 and 11)) + from system.query_log + where + event_date >= yesterday() and + current_database = currentDatabase() and + query = 'drop table data_01810 settings log_queries=1;' and + type = 'QueryFinish' + format Null; """ # ReplicatedMergeTree $CLICKHOUSE_CLIENT -nm -q """ use ordinary_$CLICKHOUSE_DATABASE; drop table if exists rep_data_01810; - create table rep_data_01810 (key Int) Engine=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rep_data_01810', '1') order by key partition by key settings max_part_removal_threads=10, concurrent_part_removal_threshold=49; - insert into rep_data_01810 select * from numbers(50); + + create table rep_data_01810 (key Int) + Engine=ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/rep_data_01810', '1') + order by key + partition by key%100 + settings max_part_removal_threads=10, concurrent_part_removal_threshold=99, min_bytes_for_wide_part=0; + + insert into rep_data_01810 select * from numbers(100); drop table rep_data_01810 settings log_queries=1; system flush logs; - select throwIf(length(thread_ids)<50) from system.query_log where event_date >= yesterday() and current_database = currentDatabase() and query = 'drop table rep_data_01810 settings log_queries=1;' and type = 'QueryFinish' format Null; + + -- sometimes the same thread can be used to remove part, due to ThreadPool, + -- hence we cannot compare strictly. + select throwIf(not(length(thread_ids) between 6 and 11)) + from system.query_log + where + event_date >= yesterday() and + current_database = currentDatabase() and + query = 'drop table rep_data_01810 settings log_queries=1;' and + type = 'QueryFinish' + format Null; """ $CLICKHOUSE_CLIENT -nm -q "drop database ordinary_$CLICKHOUSE_DATABASE" diff --git a/tests/queries/0_stateless/01852_cast_operator_4.reference b/tests/queries/0_stateless/01852_cast_operator_4.reference new file mode 100644 index 00000000000..ecfe62b1db4 --- /dev/null +++ b/tests/queries/0_stateless/01852_cast_operator_4.reference @@ -0,0 +1,19 @@ +3 +SELECT CAST([3, 4, 5][1], \'Int32\') +4 +SELECT CAST(CAST(\'[3,4,5]\', \'Array(Int64)\')[2], \'Int8\') +0 +1 +2 +SELECT CAST(CAST(\'[1,2,3]\', \'Array(UInt64)\')[CAST(CAST([number, number], \'Array(UInt8)\')[number], \'UInt64\')], \'UInt8\') +FROM numbers(3) +3 +WITH [3, 4, 5] AS x +SELECT CAST(x[1], \'Int32\') +3 +SELECT CAST((3, 4, 5).1, \'Int32\') +4 +SELECT CAST(CAST((3, 4, 5), \'Tuple(UInt64, UInt64, UInt64)\').1, \'Int32\') +3 +WITH (3, 4, 5) AS x +SELECT CAST(x.1, \'Int32\') diff --git a/tests/queries/0_stateless/01852_cast_operator_4.sql b/tests/queries/0_stateless/01852_cast_operator_4.sql new file mode 100644 index 00000000000..5c33191222b --- /dev/null +++ b/tests/queries/0_stateless/01852_cast_operator_4.sql @@ -0,0 +1,20 @@ +SELECT [3,4,5][1]::Int32; +EXPLAIN SYNTAX SELECT [3,4,5][1]::Int32; + +SELECT [3,4,5]::Array(Int64)[2]::Int8; +EXPLAIN SYNTAX SELECT [3,4,5]::Array(Int64)[2]::Int8; + +SELECT [1,2,3]::Array(UInt64)[[number, number]::Array(UInt8)[number]::UInt64]::UInt8 from numbers(3); +EXPLAIN SYNTAX SELECT [1,2,3]::Array(UInt64)[[number, number]::Array(UInt8)[number]::UInt64]::UInt8 from numbers(3); + +WITH [3,4,5] AS x SELECT x[1]::Int32; +EXPLAIN SYNTAX WITH [3,4,5] AS x SELECT x[1]::Int32; + +SELECT tuple(3,4,5).1::Int32; +EXPLAIN SYNTAX SELECT tuple(3,4,5).1::Int32; + +SELECT tuple(3,4,5)::Tuple(UInt64, UInt64, UInt64).2::Int32; +EXPLAIN SYNTAX SELECT tuple(3,4,5)::Tuple(UInt64, UInt64, UInt64).1::Int32; + +WITH tuple(3,4,5) AS x SELECT x.1::Int32; +EXPLAIN SYNTAX WITH tuple(3,4,5) AS x SELECT x.1::Int32; diff --git a/tests/queries/0_stateless/01852_map_combinator.sql b/tests/queries/0_stateless/01852_map_combinator.sql index 20923460eb6..3036e2e0ea4 100644 --- a/tests/queries/0_stateless/01852_map_combinator.sql +++ b/tests/queries/0_stateless/01852_map_combinator.sql @@ -6,7 +6,7 @@ CREATE TABLE map_comb(a int, statusMap Map(UInt16, UInt32)) ENGINE = Log; INSERT INTO map_comb VALUES (1, map(1, 10, 2, 10, 3, 10)),(1, map(3, 10, 4, 10, 5, 10)),(2, map(4, 10, 5, 10, 6, 10)),(2, map(6, 10, 7, 10, 8, 10)),(3, map(1, 10, 2, 10, 3, 10)),(4, map(3, 10, 4, 10, 5, 10)),(5, map(4, 10, 5, 10, 6, 10)),(5, map(6, 10, 7, 10, 8, 10)); -SELECT * FROM map_comb ORDER BY a; +SELECT * FROM map_comb ORDER BY a, statusMap; SELECT toTypeName(res), sumMap(statusMap) as res FROM map_comb; SELECT toTypeName(res), sumWithOverflowMap(statusMap) as res FROM map_comb; SELECT toTypeName(res), sumMapMerge(s) as res FROM (SELECT sumMapState(statusMap) AS s FROM map_comb); diff --git a/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 b/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 index a51f4c856f3..1704fedb92b 100644 --- a/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 +++ b/tests/queries/0_stateless/01881_join_on_conditions_merge.sql.j2 @@ -24,9 +24,9 @@ SET join_algorithm = 'partial_merge'; SELECT '-- partial_merge --'; SELECT '--'; -SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2; +SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 ORDER BY t1.key, t1.key2; SELECT '--'; -SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2; +SELECT t1.key, t1.key2 FROM t1 INNER ALL JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2 ORDER BY t1.key, t1.key2; SELECT '--'; SELECT t1.key FROM t1 INNER ANY JOIN t2 ON t1.id == t2.id AND t2.key == t2.key2 AND t1.key == t1.key2; diff --git a/tests/queries/0_stateless/01881_total_bytes_storage_buffer.reference b/tests/queries/0_stateless/01881_total_bytes_storage_buffer.reference index eaa9e9259df..aaa6a22d787 100644 --- a/tests/queries/0_stateless/01881_total_bytes_storage_buffer.reference +++ b/tests/queries/0_stateless/01881_total_bytes_storage_buffer.reference @@ -1,5 +1,5 @@ 0 8192 -32 +0 8192 -32 +0 diff --git a/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh b/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh index 07988da12c2..591305866a6 100755 --- a/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh +++ b/tests/queries/0_stateless/01921_concurrent_ttl_and_normal_merges_zookeeper_long.sh @@ -74,5 +74,4 @@ $CLICKHOUSE_CLIENT --query "SELECT COUNT() > 0 FROM system.part_log where table for i in $(seq 1 $NUM_REPLICAS); do $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS ttl_table$i" & done - wait diff --git a/tests/queries/0_stateless/01925_map_populate_series_on_map.reference b/tests/queries/0_stateless/01925_map_populate_series_on_map.reference index fd3d3b2450d..318f5ced231 100644 --- a/tests/queries/0_stateless/01925_map_populate_series_on_map.reference +++ b/tests/queries/0_stateless/01925_map_populate_series_on_map.reference @@ -20,7 +20,12 @@ select mapPopulateSeries(m, toUInt64(10)) from map_test; {1:1,2:0,3:2,4:0,5:0,6:0,7:0,8:0,9:0,10:0} {1:1,2:0,3:0,4:2,5:0,6:0,7:0,8:0,9:0,10:0} {1:1,2:0,3:0,4:0,5:2,6:0,7:0,8:0,9:0,10:0} -select mapPopulateSeries(m, 1000) from map_test; -- { serverError 43 } +select mapPopulateSeries(m, 10) from map_test; +{1:1,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0,10:0} +{1:1,2:2,3:0,4:0,5:0,6:0,7:0,8:0,9:0,10:0} +{1:1,2:0,3:2,4:0,5:0,6:0,7:0,8:0,9:0,10:0} +{1:1,2:0,3:0,4:2,5:0,6:0,7:0,8:0,9:0,10:0} +{1:1,2:0,3:0,4:0,5:2,6:0,7:0,8:0,9:0,10:0} select mapPopulateSeries(m, n) from map_test; {1:1,2:0} {1:1,2:2,3:0} diff --git a/tests/queries/0_stateless/01925_map_populate_series_on_map.sql b/tests/queries/0_stateless/01925_map_populate_series_on_map.sql index ac78280ec1d..635fba37cc8 100644 --- a/tests/queries/0_stateless/01925_map_populate_series_on_map.sql +++ b/tests/queries/0_stateless/01925_map_populate_series_on_map.sql @@ -6,7 +6,7 @@ create table map_test engine=TinyLog() as (select (number + 1) as n, map(1, 1, n select mapPopulateSeries(m) from map_test; select mapPopulateSeries(m, toUInt64(3)) from map_test; select mapPopulateSeries(m, toUInt64(10)) from map_test; -select mapPopulateSeries(m, 1000) from map_test; -- { serverError 43 } +select mapPopulateSeries(m, 10) from map_test; select mapPopulateSeries(m, n) from map_test; drop table map_test; diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.reference b/tests/queries/0_stateless/02006_test_positional_arguments.reference index 5fc070ffd0b..c5c5f115b0a 100644 --- a/tests/queries/0_stateless/02006_test_positional_arguments.reference +++ b/tests/queries/0_stateless/02006_test_positional_arguments.reference @@ -15,29 +15,23 @@ select x3, x2, x1 from test order by x3 desc; 100 10 1 10 1 10 1 100 100 -insert into test values (1, 10, 200), (10, 1, 200), (100, 100, 1); +insert into test values (1, 10, 100), (10, 1, 10), (100, 100, 1); select x3, x2 from test group by x3, x2; -200 1 10 1 -200 10 1 100 100 10 select x3, x2 from test group by 1, 2; -200 1 10 1 -200 10 1 100 100 10 select x1, x2, x3 from test order by x3 limit 1 by x3; 100 100 1 10 1 10 1 10 100 -1 10 200 select x1, x2, x3 from test order by 3 limit 1 by 3; 100 100 1 10 1 10 1 10 100 -1 10 200 select x1, x2, x3 from test order by x3 limit 1 by x1; 100 100 1 10 1 10 diff --git a/tests/queries/0_stateless/02006_test_positional_arguments.sql b/tests/queries/0_stateless/02006_test_positional_arguments.sql index 3a2cf76f6c4..54b55c4a9f8 100644 --- a/tests/queries/0_stateless/02006_test_positional_arguments.sql +++ b/tests/queries/0_stateless/02006_test_positional_arguments.sql @@ -13,7 +13,7 @@ select x3, x2, x1 from test order by x3; select x3, x2, x1 from test order by 1 desc; select x3, x2, x1 from test order by x3 desc; -insert into test values (1, 10, 200), (10, 1, 200), (100, 100, 1); +insert into test values (1, 10, 100), (10, 1, 10), (100, 100, 1); select x3, x2 from test group by x3, x2; select x3, x2 from test group by 1, 2; diff --git a/tests/queries/0_stateless/02006_use_constants_in_with_and_select.sql b/tests/queries/0_stateless/02006_use_constants_in_with_and_select.sql index 91171c9ab7b..daca6c5d0c7 100644 --- a/tests/queries/0_stateless/02006_use_constants_in_with_and_select.sql +++ b/tests/queries/0_stateless/02006_use_constants_in_with_and_select.sql @@ -2,20 +2,20 @@ SELECT 1 AS max_size, groupArray(max_size)(col) FROM - ( + (SELECT col FROM ( SELECT 1 AS col UNION ALL SELECT 2 - ); + ) ORDER BY col); WITH 1 AS max_size SELECT groupArray(max_size)(col) FROM - ( + (SELECT col FROM ( SELECT 1 as col UNION ALL SELECT 2 - ); + ) ORDER BY col); WITH 0.1 AS level SELECT quantile(level)(number) diff --git a/tests/queries/0_stateless/02021_create_database_with_comment.reference b/tests/queries/0_stateless/02021_create_database_with_comment.reference index 65fc9b6d290..693ea584b84 100644 --- a/tests/queries/0_stateless/02021_create_database_with_comment.reference +++ b/tests/queries/0_stateless/02021_create_database_with_comment.reference @@ -1,5 +1,5 @@ engine : Memory -CREATE DATABASE default\nENGINE = Memory()\nCOMMENT \'Test DB with comment\' +CREATE DATABASE default\nENGINE = Memory\nCOMMENT \'Test DB with comment\' comment= Test DB with comment engine : Atomic diff --git a/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.reference b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh new file mode 100755 index 00000000000..7e8579f7cbe --- /dev/null +++ b/tests/queries/0_stateless/02044_url_glob_parallel_connection_refused.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Tags: distributed + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +i=0 retries=5 +# Connecting to wrong address and checking for race condition +while [[ $i -lt $retries ]]; do + timeout 5s ${CLICKHOUSE_CLIENT} --max_threads 10 --query "SELECT * FROM url('http://128.0.0.{1..10}:${CLICKHOUSE_PORT_HTTP}/?query=SELECT+sleep(1)', TSV, 'x UInt8')" --format Null 2>/dev/null + ((++i)) +done diff --git a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference new file mode 100644 index 00000000000..da7b788b157 --- /dev/null +++ b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.reference @@ -0,0 +1,18 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh new file mode 100755 index 00000000000..0e7a9f17225 --- /dev/null +++ b/tests/queries/0_stateless/02100_multiple_hosts_command_line_set.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# default values test +${CLICKHOUSE_CLIENT} --query "SELECT 1" + +# backward compatibility test +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; + +not_resolvable_host="notlocalhost" +exception_msg="Cannot resolve host (${not_resolvable_host}), error 0: ${not_resolvable_host}. +Code: 198. DB::Exception: Not found address of host: ${not_resolvable_host}. (DNS_ERROR) +" +error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --host "${not_resolvable_host}" --query "SELECT 1" 2>&1 > /dev/null)"; +[ "${error}" == "${exception_msg}" ]; echo "$?" + +not_number_port="abc" +exception_msg="Bad arguments: the argument ('${CLICKHOUSE_HOST}:${not_number_port}') for option '--host' is invalid." +error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_number_port}" --query "SELECT 1" 2>&1 > /dev/null)"; +[ "${error}" == "${exception_msg}" ]; echo "$?" + +not_alive_host="10.100.0.0" +${CLICKHOUSE_CLIENT} --host "${not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; + +not_alive_port="1" +exception_msg="Code: 210. DB::NetException: Connection refused (${CLICKHOUSE_HOST}:${not_alive_port}). (NETWORK_ERROR) +" +error="$(${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_alive_port}" --query "SELECT 1" 2>&1 > /dev/null)" +[ "${error}" == "${exception_msg}" ]; echo "$?" +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${not_alive_port}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --port "${CLICKHOUSE_PORT_TCP}" --port "${not_alive_port}" --query "SELECT 1"; + +ipv6_host_without_brackets="2001:3984:3989::1:1000" +exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_without_brackets}). (NETWORK_ERROR) +" +error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_without_brackets}" --query "SELECT 1" 2>&1 > /dev/null)" +[ "${error}" == "${exception_msg}" ]; echo "$?" + +ipv6_host_with_brackets="[2001:3984:3989::1:1000]" +exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_with_brackets}). (NETWORK_ERROR) +" +error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_with_brackets}" --query "SELECT 1" 2>&1 > /dev/null)" +[ "${error}" == "${exception_msg}" ]; echo "$?" + +exception_msg="Code: 210. DB::NetException: Connection refused (${ipv6_host_with_brackets}:${not_alive_port}). (NETWORK_ERROR) +" +error="$(${CLICKHOUSE_CLIENT} --host "${ipv6_host_with_brackets}" --port "${not_alive_port}" --query "SELECT 1" 2>&1 > /dev/null)" +[ "${error}" == "${exception_msg}" ]; echo "$?" + + +${CLICKHOUSE_CLIENT} --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "${CLICKHOUSE_HOST}" --host "{$not_alive_host}" --port "${CLICKHOUSE_PORT_TCP}" --query "SELECT 1"; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --host "{$not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1" 2> /dev/null; +${CLICKHOUSE_CLIENT} --port "${CLICKHOUSE_PORT_TCP}" --port "${CLICKHOUSE_PORT_TCP}" --port "${CLICKHOUSE_PORT_TCP}" --host "{$not_alive_host}" --host "${CLICKHOUSE_HOST}" --query "SELECT 1"; + diff --git a/tests/queries/0_stateless/02104_overcommit_memory.reference b/tests/queries/0_stateless/02104_overcommit_memory.reference new file mode 100644 index 00000000000..b108f48e0fa --- /dev/null +++ b/tests/queries/0_stateless/02104_overcommit_memory.reference @@ -0,0 +1 @@ +OVERCOMMITED WITH USER LIMIT WAS KILLED diff --git a/tests/queries/0_stateless/02104_overcommit_memory.sh b/tests/queries/0_stateless/02104_overcommit_memory.sh new file mode 100755 index 00000000000..140557304c6 --- /dev/null +++ b/tests/queries/0_stateless/02104_overcommit_memory.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q 'CREATE USER IF NOT EXISTS u1 IDENTIFIED WITH no_password' +$CLICKHOUSE_CLIENT -q 'GRANT ALL ON *.* TO u1' + +function overcommited() +{ + while true; do + $CLICKHOUSE_CLIENT -u u1 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_guaranteed_memory_usage=1,memory_usage_overcommit_max_wait_microseconds=500' 2>&1 | grep -F -q "MEMORY_LIMIT_EXCEEDED" && echo "OVERCOMMITED WITH USER LIMIT IS KILLED" + done +} + +function expect_execution() +{ + while true; do + $CLICKHOUSE_CLIENT -u u1 -q 'SELECT number FROM numbers(130000) GROUP BY number SETTINGS max_memory_usage_for_user=5000000,max_guaranteed_memory_usage=2,memory_usage_overcommit_max_wait_microseconds=500' >/dev/null 2>/dev/null + done +} + +export -f overcommited +export -f expect_execution + +function user_test() +{ + for _ in {1..10}; + do + timeout 10 bash -c overcommited & + timeout 10 bash -c expect_execution & + done; + + wait +} + +output=$(user_test) + +if test -z "$output" +then + echo "OVERCOMMITED WITH USER LIMIT WAS NOT KILLED" +else + echo "OVERCOMMITED WITH USER LIMIT WAS KILLED" +fi + +$CLICKHOUSE_CLIENT -q 'DROP USER IF EXISTS u1' diff --git a/tests/queries/0_stateless/02117_show_create_table_system.reference b/tests/queries/0_stateless/02117_show_create_table_system.reference index 234804f1078..678fe35fd96 100644 --- a/tests/queries/0_stateless/02117_show_create_table_system.reference +++ b/tests/queries/0_stateless/02117_show_create_table_system.reference @@ -1,5 +1,5 @@ CREATE TABLE system.aggregate_function_combinators\n(\n `name` String,\n `is_internal` UInt8\n)\nENGINE = SystemAggregateFunctionCombinators()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' -CREATE TABLE system.asynchronous_inserts\n(\n `query` String,\n `database` String,\n `table` String,\n `format` String,\n `first_update` DateTime64(6),\n `last_update` DateTime64(6),\n `total_bytes` UInt64,\n `entries.query_id` Array(String),\n `entries.bytes` Array(UInt64),\n `entries.finished` Array(UInt8),\n `entries.exception` Array(String)\n)\nENGINE = AsynchronousInserts()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' +CREATE TABLE system.asynchronous_inserts\n(\n `query` String,\n `database` String,\n `table` String,\n `format` String,\n `first_update` DateTime64(6),\n `last_update` DateTime64(6),\n `total_bytes` UInt64,\n `entries.query_id` Array(String),\n `entries.bytes` Array(UInt64),\n `entries.finished` Array(UInt8),\n `entries.exception` Array(String)\n)\nENGINE = SystemAsynchronousInserts()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.asynchronous_metrics\n(\n `metric` String,\n `value` Float64\n)\nENGINE = SystemAsynchronousMetrics()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.build_options\n(\n `name` String,\n `value` String\n)\nENGINE = SystemBuildOptions()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.clusters\n(\n `cluster` String,\n `shard_num` UInt32,\n `shard_weight` UInt32,\n `replica_num` UInt32,\n `host_name` String,\n `host_address` String,\n `port` UInt16,\n `is_local` UInt8,\n `user` String,\n `default_database` String,\n `errors_count` UInt32,\n `slowdowns_count` UInt32,\n `estimated_recovery_time` UInt32\n)\nENGINE = SystemClusters()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' @@ -20,7 +20,7 @@ CREATE TABLE system.errors\n(\n `name` String,\n `code` Int32,\n `value CREATE TABLE system.events\n(\n `event` String,\n `value` UInt64,\n `description` String\n)\nENGINE = SystemEvents()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.formats\n(\n `name` String,\n `is_input` UInt8,\n `is_output` UInt8\n)\nENGINE = SystemFormats()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.functions\n(\n `name` String,\n `is_aggregate` UInt8,\n `case_insensitive` UInt8,\n `alias_to` String,\n `create_query` String,\n `origin` Enum8(\'System\' = 0, \'SQLUserDefined\' = 1, \'ExecutableUserDefined\' = 2)\n)\nENGINE = SystemFunctions()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' -CREATE TABLE system.grants\n(\n `user_name` Nullable(String),\n `role_name` Nullable(String),\n `access_type` Enum8(\'POSTGRES\' = -128, \'SQLITE\' = -127, \'ODBC\' = -126, \'JDBC\' = -125, \'HDFS\' = -124, \'S3\' = -123, \'SOURCES\' = -122, \'ALL\' = -121, \'NONE\' = -120, \'SHOW DATABASES\' = 0, \'SHOW TABLES\' = 1, \'SHOW COLUMNS\' = 2, \'SHOW DICTIONARIES\' = 3, \'SHOW\' = 4, \'SELECT\' = 5, \'INSERT\' = 6, \'ALTER UPDATE\' = 7, \'ALTER DELETE\' = 8, \'ALTER ADD COLUMN\' = 9, \'ALTER MODIFY COLUMN\' = 10, \'ALTER DROP COLUMN\' = 11, \'ALTER COMMENT COLUMN\' = 12, \'ALTER CLEAR COLUMN\' = 13, \'ALTER RENAME COLUMN\' = 14, \'ALTER MATERIALIZE COLUMN\' = 15, \'ALTER COLUMN\' = 16, \'ALTER MODIFY COMMENT\' = 17, \'ALTER ORDER BY\' = 18, \'ALTER SAMPLE BY\' = 19, \'ALTER ADD INDEX\' = 20, \'ALTER DROP INDEX\' = 21, \'ALTER MATERIALIZE INDEX\' = 22, \'ALTER CLEAR INDEX\' = 23, \'ALTER INDEX\' = 24, \'ALTER ADD PROJECTION\' = 25, \'ALTER DROP PROJECTION\' = 26, \'ALTER MATERIALIZE PROJECTION\' = 27, \'ALTER CLEAR PROJECTION\' = 28, \'ALTER PROJECTION\' = 29, \'ALTER ADD CONSTRAINT\' = 30, \'ALTER DROP CONSTRAINT\' = 31, \'ALTER CONSTRAINT\' = 32, \'ALTER TTL\' = 33, \'ALTER MATERIALIZE TTL\' = 34, \'ALTER SETTINGS\' = 35, \'ALTER MOVE PARTITION\' = 36, \'ALTER FETCH PARTITION\' = 37, \'ALTER FREEZE PARTITION\' = 38, \'ALTER DATABASE SETTINGS\' = 39, \'ALTER TABLE\' = 40, \'ALTER DATABASE\' = 41, \'ALTER VIEW REFRESH\' = 42, \'ALTER VIEW MODIFY QUERY\' = 43, \'ALTER VIEW\' = 44, \'ALTER\' = 45, \'CREATE DATABASE\' = 46, \'CREATE TABLE\' = 47, \'CREATE VIEW\' = 48, \'CREATE DICTIONARY\' = 49, \'CREATE TEMPORARY TABLE\' = 50, \'CREATE FUNCTION\' = 51, \'CREATE\' = 52, \'DROP DATABASE\' = 53, \'DROP TABLE\' = 54, \'DROP VIEW\' = 55, \'DROP DICTIONARY\' = 56, \'DROP FUNCTION\' = 57, \'DROP\' = 58, \'TRUNCATE\' = 59, \'OPTIMIZE\' = 60, \'KILL QUERY\' = 61, \'MOVE PARTITION BETWEEN SHARDS\' = 62, \'CREATE USER\' = 63, \'ALTER USER\' = 64, \'DROP USER\' = 65, \'CREATE ROLE\' = 66, \'ALTER ROLE\' = 67, \'DROP ROLE\' = 68, \'ROLE ADMIN\' = 69, \'CREATE ROW POLICY\' = 70, \'ALTER ROW POLICY\' = 71, \'DROP ROW POLICY\' = 72, \'CREATE QUOTA\' = 73, \'ALTER QUOTA\' = 74, \'DROP QUOTA\' = 75, \'CREATE SETTINGS PROFILE\' = 76, \'ALTER SETTINGS PROFILE\' = 77, \'DROP SETTINGS PROFILE\' = 78, \'SHOW USERS\' = 79, \'SHOW ROLES\' = 80, \'SHOW ROW POLICIES\' = 81, \'SHOW QUOTAS\' = 82, \'SHOW SETTINGS PROFILES\' = 83, \'SHOW ACCESS\' = 84, \'ACCESS MANAGEMENT\' = 85, \'SYSTEM SHUTDOWN\' = 86, \'SYSTEM DROP DNS CACHE\' = 87, \'SYSTEM DROP MARK CACHE\' = 88, \'SYSTEM DROP UNCOMPRESSED CACHE\' = 89, \'SYSTEM DROP MMAP CACHE\' = 90, \'SYSTEM DROP COMPILED EXPRESSION CACHE\' = 91, \'SYSTEM DROP CACHE\' = 92, \'SYSTEM RELOAD CONFIG\' = 93, \'SYSTEM RELOAD SYMBOLS\' = 94, \'SYSTEM RELOAD DICTIONARY\' = 95, \'SYSTEM RELOAD MODEL\' = 96, \'SYSTEM RELOAD FUNCTION\' = 97, \'SYSTEM RELOAD EMBEDDED DICTIONARIES\' = 98, \'SYSTEM RELOAD\' = 99, \'SYSTEM RESTART DISK\' = 100, \'SYSTEM MERGES\' = 101, \'SYSTEM TTL MERGES\' = 102, \'SYSTEM FETCHES\' = 103, \'SYSTEM MOVES\' = 104, \'SYSTEM DISTRIBUTED SENDS\' = 105, \'SYSTEM REPLICATED SENDS\' = 106, \'SYSTEM SENDS\' = 107, \'SYSTEM REPLICATION QUEUES\' = 108, \'SYSTEM DROP REPLICA\' = 109, \'SYSTEM SYNC REPLICA\' = 110, \'SYSTEM RESTART REPLICA\' = 111, \'SYSTEM RESTORE REPLICA\' = 112, \'SYSTEM FLUSH DISTRIBUTED\' = 113, \'SYSTEM FLUSH LOGS\' = 114, \'SYSTEM FLUSH\' = 115, \'SYSTEM THREAD FUZZER\' = 116, \'SYSTEM\' = 117, \'dictGet\' = 118, \'addressToLine\' = 119, \'addressToSymbol\' = 120, \'demangle\' = 121, \'INTROSPECTION\' = 122, \'FILE\' = 123, \'URL\' = 124, \'REMOTE\' = 125, \'MONGO\' = 126, \'MYSQL\' = 127),\n `database` Nullable(String),\n `table` Nullable(String),\n `column` Nullable(String),\n `is_partial_revoke` UInt8,\n `grant_option` UInt8\n)\nENGINE = SystemGrants()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' +CREATE TABLE system.grants\n(\n `user_name` Nullable(String),\n `role_name` Nullable(String),\n `access_type` Enum16(\'SHOW DATABASES\' = 0, \'SHOW TABLES\' = 1, \'SHOW COLUMNS\' = 2, \'SHOW DICTIONARIES\' = 3, \'SHOW\' = 4, \'SELECT\' = 5, \'INSERT\' = 6, \'ALTER UPDATE\' = 7, \'ALTER DELETE\' = 8, \'ALTER ADD COLUMN\' = 9, \'ALTER MODIFY COLUMN\' = 10, \'ALTER DROP COLUMN\' = 11, \'ALTER COMMENT COLUMN\' = 12, \'ALTER CLEAR COLUMN\' = 13, \'ALTER RENAME COLUMN\' = 14, \'ALTER MATERIALIZE COLUMN\' = 15, \'ALTER COLUMN\' = 16, \'ALTER MODIFY COMMENT\' = 17, \'ALTER ORDER BY\' = 18, \'ALTER SAMPLE BY\' = 19, \'ALTER ADD INDEX\' = 20, \'ALTER DROP INDEX\' = 21, \'ALTER MATERIALIZE INDEX\' = 22, \'ALTER CLEAR INDEX\' = 23, \'ALTER INDEX\' = 24, \'ALTER ADD PROJECTION\' = 25, \'ALTER DROP PROJECTION\' = 26, \'ALTER MATERIALIZE PROJECTION\' = 27, \'ALTER CLEAR PROJECTION\' = 28, \'ALTER PROJECTION\' = 29, \'ALTER ADD CONSTRAINT\' = 30, \'ALTER DROP CONSTRAINT\' = 31, \'ALTER CONSTRAINT\' = 32, \'ALTER TTL\' = 33, \'ALTER MATERIALIZE TTL\' = 34, \'ALTER SETTINGS\' = 35, \'ALTER MOVE PARTITION\' = 36, \'ALTER FETCH PARTITION\' = 37, \'ALTER FREEZE PARTITION\' = 38, \'ALTER DATABASE SETTINGS\' = 39, \'ALTER TABLE\' = 40, \'ALTER DATABASE\' = 41, \'ALTER VIEW REFRESH\' = 42, \'ALTER VIEW MODIFY QUERY\' = 43, \'ALTER VIEW\' = 44, \'ALTER\' = 45, \'CREATE DATABASE\' = 46, \'CREATE TABLE\' = 47, \'CREATE VIEW\' = 48, \'CREATE DICTIONARY\' = 49, \'CREATE TEMPORARY TABLE\' = 50, \'CREATE FUNCTION\' = 51, \'CREATE\' = 52, \'DROP DATABASE\' = 53, \'DROP TABLE\' = 54, \'DROP VIEW\' = 55, \'DROP DICTIONARY\' = 56, \'DROP FUNCTION\' = 57, \'DROP\' = 58, \'TRUNCATE\' = 59, \'OPTIMIZE\' = 60, \'KILL QUERY\' = 61, \'MOVE PARTITION BETWEEN SHARDS\' = 62, \'CREATE USER\' = 63, \'ALTER USER\' = 64, \'DROP USER\' = 65, \'CREATE ROLE\' = 66, \'ALTER ROLE\' = 67, \'DROP ROLE\' = 68, \'ROLE ADMIN\' = 69, \'CREATE ROW POLICY\' = 70, \'ALTER ROW POLICY\' = 71, \'DROP ROW POLICY\' = 72, \'CREATE QUOTA\' = 73, \'ALTER QUOTA\' = 74, \'DROP QUOTA\' = 75, \'CREATE SETTINGS PROFILE\' = 76, \'ALTER SETTINGS PROFILE\' = 77, \'DROP SETTINGS PROFILE\' = 78, \'SHOW USERS\' = 79, \'SHOW ROLES\' = 80, \'SHOW ROW POLICIES\' = 81, \'SHOW QUOTAS\' = 82, \'SHOW SETTINGS PROFILES\' = 83, \'SHOW ACCESS\' = 84, \'ACCESS MANAGEMENT\' = 85, \'SYSTEM SHUTDOWN\' = 86, \'SYSTEM DROP DNS CACHE\' = 87, \'SYSTEM DROP MARK CACHE\' = 88, \'SYSTEM DROP UNCOMPRESSED CACHE\' = 89, \'SYSTEM DROP MMAP CACHE\' = 90, \'SYSTEM DROP COMPILED EXPRESSION CACHE\' = 91, \'SYSTEM DROP CACHE\' = 92, \'SYSTEM RELOAD CONFIG\' = 93, \'SYSTEM RELOAD SYMBOLS\' = 94, \'SYSTEM RELOAD DICTIONARY\' = 95, \'SYSTEM RELOAD MODEL\' = 96, \'SYSTEM RELOAD FUNCTION\' = 97, \'SYSTEM RELOAD EMBEDDED DICTIONARIES\' = 98, \'SYSTEM RELOAD\' = 99, \'SYSTEM RESTART DISK\' = 100, \'SYSTEM MERGES\' = 101, \'SYSTEM TTL MERGES\' = 102, \'SYSTEM FETCHES\' = 103, \'SYSTEM MOVES\' = 104, \'SYSTEM DISTRIBUTED SENDS\' = 105, \'SYSTEM REPLICATED SENDS\' = 106, \'SYSTEM SENDS\' = 107, \'SYSTEM REPLICATION QUEUES\' = 108, \'SYSTEM DROP REPLICA\' = 109, \'SYSTEM SYNC REPLICA\' = 110, \'SYSTEM RESTART REPLICA\' = 111, \'SYSTEM RESTORE REPLICA\' = 112, \'SYSTEM FLUSH DISTRIBUTED\' = 113, \'SYSTEM FLUSH LOGS\' = 114, \'SYSTEM FLUSH\' = 115, \'SYSTEM THREAD FUZZER\' = 116, \'SYSTEM\' = 117, \'dictGet\' = 118, \'addressToLine\' = 119, \'addressToLineWithInlines\' = 120, \'addressToSymbol\' = 121, \'demangle\' = 122, \'INTROSPECTION\' = 123, \'FILE\' = 124, \'URL\' = 125, \'REMOTE\' = 126, \'MONGO\' = 127, \'MYSQL\' = 128, \'POSTGRES\' = 129, \'SQLITE\' = 130, \'ODBC\' = 131, \'JDBC\' = 132, \'HDFS\' = 133, \'S3\' = 134, \'SOURCES\' = 135, \'ALL\' = 136, \'NONE\' = 137),\n `database` Nullable(String),\n `table` Nullable(String),\n `column` Nullable(String),\n `is_partial_revoke` UInt8,\n `grant_option` UInt8\n)\nENGINE = SystemGrants()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.graphite_retentions\n(\n `config_name` String,\n `rule_type` String,\n `regexp` String,\n `function` String,\n `age` UInt64,\n `precision` UInt64,\n `priority` UInt16,\n `is_default` UInt8,\n `Tables.database` Array(String),\n `Tables.table` Array(String)\n)\nENGINE = SystemGraphite()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.licenses\n(\n `library_name` String,\n `license_type` String,\n `license_path` String,\n `license_text` String\n)\nENGINE = SystemLicenses()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.macros\n(\n `macro` String,\n `substitution` String\n)\nENGINE = SystemMacros()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' @@ -35,7 +35,7 @@ CREATE TABLE system.one\n(\n `dummy` UInt8\n)\nENGINE = SystemOne()\nCOMMENT CREATE TABLE system.part_moves_between_shards\n(\n `database` String,\n `table` String,\n `task_name` String,\n `task_uuid` UUID,\n `create_time` DateTime,\n `part_name` String,\n `part_uuid` UUID,\n `to_shard` String,\n `dst_part_name` String,\n `update_time` DateTime,\n `state` String,\n `rollback` UInt8,\n `num_tries` UInt32,\n `last_exception` String\n)\nENGINE = SystemShardMoves()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.parts\n(\n `partition` String,\n `name` String,\n `uuid` UUID,\n `part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `secondary_indices_compressed_bytes` UInt64,\n `secondary_indices_uncompressed_bytes` UInt64,\n `secondary_indices_marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `is_frozen` UInt8,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `hash_of_all_files` String,\n `hash_of_uncompressed_files` String,\n `uncompressed_hash_of_compressed_files` String,\n `delete_ttl_info_min` DateTime,\n `delete_ttl_info_max` DateTime,\n `move_ttl_info.expression` Array(String),\n `move_ttl_info.min` Array(DateTime),\n `move_ttl_info.max` Array(DateTime),\n `default_compression_codec` String,\n `recompression_ttl_info.expression` Array(String),\n `recompression_ttl_info.min` Array(DateTime),\n `recompression_ttl_info.max` Array(DateTime),\n `group_by_ttl_info.expression` Array(String),\n `group_by_ttl_info.min` Array(DateTime),\n `group_by_ttl_info.max` Array(DateTime),\n `rows_where_ttl_info.expression` Array(String),\n `rows_where_ttl_info.min` Array(DateTime),\n `rows_where_ttl_info.max` Array(DateTime),\n `projections` Array(String),\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemParts()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.parts_columns\n(\n `partition` String,\n `name` String,\n `uuid` UUID,\n `part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `column` String,\n `type` String,\n `column_position` UInt64,\n `default_kind` String,\n `default_expression` String,\n `column_bytes_on_disk` UInt64,\n `column_data_compressed_bytes` UInt64,\n `column_data_uncompressed_bytes` UInt64,\n `column_marks_bytes` UInt64,\n `serialization_kind` String,\n `subcolumns.names` Array(String),\n `subcolumns.types` Array(String),\n `subcolumns.serializations` Array(String),\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemPartsColumns()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' -CREATE TABLE system.privileges\n(\n `privilege` Enum8(\'POSTGRES\' = -128, \'SQLITE\' = -127, \'ODBC\' = -126, \'JDBC\' = -125, \'HDFS\' = -124, \'S3\' = -123, \'SOURCES\' = -122, \'ALL\' = -121, \'NONE\' = -120, \'SHOW DATABASES\' = 0, \'SHOW TABLES\' = 1, \'SHOW COLUMNS\' = 2, \'SHOW DICTIONARIES\' = 3, \'SHOW\' = 4, \'SELECT\' = 5, \'INSERT\' = 6, \'ALTER UPDATE\' = 7, \'ALTER DELETE\' = 8, \'ALTER ADD COLUMN\' = 9, \'ALTER MODIFY COLUMN\' = 10, \'ALTER DROP COLUMN\' = 11, \'ALTER COMMENT COLUMN\' = 12, \'ALTER CLEAR COLUMN\' = 13, \'ALTER RENAME COLUMN\' = 14, \'ALTER MATERIALIZE COLUMN\' = 15, \'ALTER COLUMN\' = 16, \'ALTER MODIFY COMMENT\' = 17, \'ALTER ORDER BY\' = 18, \'ALTER SAMPLE BY\' = 19, \'ALTER ADD INDEX\' = 20, \'ALTER DROP INDEX\' = 21, \'ALTER MATERIALIZE INDEX\' = 22, \'ALTER CLEAR INDEX\' = 23, \'ALTER INDEX\' = 24, \'ALTER ADD PROJECTION\' = 25, \'ALTER DROP PROJECTION\' = 26, \'ALTER MATERIALIZE PROJECTION\' = 27, \'ALTER CLEAR PROJECTION\' = 28, \'ALTER PROJECTION\' = 29, \'ALTER ADD CONSTRAINT\' = 30, \'ALTER DROP CONSTRAINT\' = 31, \'ALTER CONSTRAINT\' = 32, \'ALTER TTL\' = 33, \'ALTER MATERIALIZE TTL\' = 34, \'ALTER SETTINGS\' = 35, \'ALTER MOVE PARTITION\' = 36, \'ALTER FETCH PARTITION\' = 37, \'ALTER FREEZE PARTITION\' = 38, \'ALTER DATABASE SETTINGS\' = 39, \'ALTER TABLE\' = 40, \'ALTER DATABASE\' = 41, \'ALTER VIEW REFRESH\' = 42, \'ALTER VIEW MODIFY QUERY\' = 43, \'ALTER VIEW\' = 44, \'ALTER\' = 45, \'CREATE DATABASE\' = 46, \'CREATE TABLE\' = 47, \'CREATE VIEW\' = 48, \'CREATE DICTIONARY\' = 49, \'CREATE TEMPORARY TABLE\' = 50, \'CREATE FUNCTION\' = 51, \'CREATE\' = 52, \'DROP DATABASE\' = 53, \'DROP TABLE\' = 54, \'DROP VIEW\' = 55, \'DROP DICTIONARY\' = 56, \'DROP FUNCTION\' = 57, \'DROP\' = 58, \'TRUNCATE\' = 59, \'OPTIMIZE\' = 60, \'KILL QUERY\' = 61, \'MOVE PARTITION BETWEEN SHARDS\' = 62, \'CREATE USER\' = 63, \'ALTER USER\' = 64, \'DROP USER\' = 65, \'CREATE ROLE\' = 66, \'ALTER ROLE\' = 67, \'DROP ROLE\' = 68, \'ROLE ADMIN\' = 69, \'CREATE ROW POLICY\' = 70, \'ALTER ROW POLICY\' = 71, \'DROP ROW POLICY\' = 72, \'CREATE QUOTA\' = 73, \'ALTER QUOTA\' = 74, \'DROP QUOTA\' = 75, \'CREATE SETTINGS PROFILE\' = 76, \'ALTER SETTINGS PROFILE\' = 77, \'DROP SETTINGS PROFILE\' = 78, \'SHOW USERS\' = 79, \'SHOW ROLES\' = 80, \'SHOW ROW POLICIES\' = 81, \'SHOW QUOTAS\' = 82, \'SHOW SETTINGS PROFILES\' = 83, \'SHOW ACCESS\' = 84, \'ACCESS MANAGEMENT\' = 85, \'SYSTEM SHUTDOWN\' = 86, \'SYSTEM DROP DNS CACHE\' = 87, \'SYSTEM DROP MARK CACHE\' = 88, \'SYSTEM DROP UNCOMPRESSED CACHE\' = 89, \'SYSTEM DROP MMAP CACHE\' = 90, \'SYSTEM DROP COMPILED EXPRESSION CACHE\' = 91, \'SYSTEM DROP CACHE\' = 92, \'SYSTEM RELOAD CONFIG\' = 93, \'SYSTEM RELOAD SYMBOLS\' = 94, \'SYSTEM RELOAD DICTIONARY\' = 95, \'SYSTEM RELOAD MODEL\' = 96, \'SYSTEM RELOAD FUNCTION\' = 97, \'SYSTEM RELOAD EMBEDDED DICTIONARIES\' = 98, \'SYSTEM RELOAD\' = 99, \'SYSTEM RESTART DISK\' = 100, \'SYSTEM MERGES\' = 101, \'SYSTEM TTL MERGES\' = 102, \'SYSTEM FETCHES\' = 103, \'SYSTEM MOVES\' = 104, \'SYSTEM DISTRIBUTED SENDS\' = 105, \'SYSTEM REPLICATED SENDS\' = 106, \'SYSTEM SENDS\' = 107, \'SYSTEM REPLICATION QUEUES\' = 108, \'SYSTEM DROP REPLICA\' = 109, \'SYSTEM SYNC REPLICA\' = 110, \'SYSTEM RESTART REPLICA\' = 111, \'SYSTEM RESTORE REPLICA\' = 112, \'SYSTEM FLUSH DISTRIBUTED\' = 113, \'SYSTEM FLUSH LOGS\' = 114, \'SYSTEM FLUSH\' = 115, \'SYSTEM THREAD FUZZER\' = 116, \'SYSTEM\' = 117, \'dictGet\' = 118, \'addressToLine\' = 119, \'addressToSymbol\' = 120, \'demangle\' = 121, \'INTROSPECTION\' = 122, \'FILE\' = 123, \'URL\' = 124, \'REMOTE\' = 125, \'MONGO\' = 126, \'MYSQL\' = 127),\n `aliases` Array(String),\n `level` Nullable(Enum8(\'GLOBAL\' = 0, \'DATABASE\' = 1, \'TABLE\' = 2, \'DICTIONARY\' = 3, \'VIEW\' = 4, \'COLUMN\' = 5)),\n `parent_group` Nullable(Enum8(\'POSTGRES\' = -128, \'SQLITE\' = -127, \'ODBC\' = -126, \'JDBC\' = -125, \'HDFS\' = -124, \'S3\' = -123, \'SOURCES\' = -122, \'ALL\' = -121, \'NONE\' = -120, \'SHOW DATABASES\' = 0, \'SHOW TABLES\' = 1, \'SHOW COLUMNS\' = 2, \'SHOW DICTIONARIES\' = 3, \'SHOW\' = 4, \'SELECT\' = 5, \'INSERT\' = 6, \'ALTER UPDATE\' = 7, \'ALTER DELETE\' = 8, \'ALTER ADD COLUMN\' = 9, \'ALTER MODIFY COLUMN\' = 10, \'ALTER DROP COLUMN\' = 11, \'ALTER COMMENT COLUMN\' = 12, \'ALTER CLEAR COLUMN\' = 13, \'ALTER RENAME COLUMN\' = 14, \'ALTER MATERIALIZE COLUMN\' = 15, \'ALTER COLUMN\' = 16, \'ALTER MODIFY COMMENT\' = 17, \'ALTER ORDER BY\' = 18, \'ALTER SAMPLE BY\' = 19, \'ALTER ADD INDEX\' = 20, \'ALTER DROP INDEX\' = 21, \'ALTER MATERIALIZE INDEX\' = 22, \'ALTER CLEAR INDEX\' = 23, \'ALTER INDEX\' = 24, \'ALTER ADD PROJECTION\' = 25, \'ALTER DROP PROJECTION\' = 26, \'ALTER MATERIALIZE PROJECTION\' = 27, \'ALTER CLEAR PROJECTION\' = 28, \'ALTER PROJECTION\' = 29, \'ALTER ADD CONSTRAINT\' = 30, \'ALTER DROP CONSTRAINT\' = 31, \'ALTER CONSTRAINT\' = 32, \'ALTER TTL\' = 33, \'ALTER MATERIALIZE TTL\' = 34, \'ALTER SETTINGS\' = 35, \'ALTER MOVE PARTITION\' = 36, \'ALTER FETCH PARTITION\' = 37, \'ALTER FREEZE PARTITION\' = 38, \'ALTER DATABASE SETTINGS\' = 39, \'ALTER TABLE\' = 40, \'ALTER DATABASE\' = 41, \'ALTER VIEW REFRESH\' = 42, \'ALTER VIEW MODIFY QUERY\' = 43, \'ALTER VIEW\' = 44, \'ALTER\' = 45, \'CREATE DATABASE\' = 46, \'CREATE TABLE\' = 47, \'CREATE VIEW\' = 48, \'CREATE DICTIONARY\' = 49, \'CREATE TEMPORARY TABLE\' = 50, \'CREATE FUNCTION\' = 51, \'CREATE\' = 52, \'DROP DATABASE\' = 53, \'DROP TABLE\' = 54, \'DROP VIEW\' = 55, \'DROP DICTIONARY\' = 56, \'DROP FUNCTION\' = 57, \'DROP\' = 58, \'TRUNCATE\' = 59, \'OPTIMIZE\' = 60, \'KILL QUERY\' = 61, \'MOVE PARTITION BETWEEN SHARDS\' = 62, \'CREATE USER\' = 63, \'ALTER USER\' = 64, \'DROP USER\' = 65, \'CREATE ROLE\' = 66, \'ALTER ROLE\' = 67, \'DROP ROLE\' = 68, \'ROLE ADMIN\' = 69, \'CREATE ROW POLICY\' = 70, \'ALTER ROW POLICY\' = 71, \'DROP ROW POLICY\' = 72, \'CREATE QUOTA\' = 73, \'ALTER QUOTA\' = 74, \'DROP QUOTA\' = 75, \'CREATE SETTINGS PROFILE\' = 76, \'ALTER SETTINGS PROFILE\' = 77, \'DROP SETTINGS PROFILE\' = 78, \'SHOW USERS\' = 79, \'SHOW ROLES\' = 80, \'SHOW ROW POLICIES\' = 81, \'SHOW QUOTAS\' = 82, \'SHOW SETTINGS PROFILES\' = 83, \'SHOW ACCESS\' = 84, \'ACCESS MANAGEMENT\' = 85, \'SYSTEM SHUTDOWN\' = 86, \'SYSTEM DROP DNS CACHE\' = 87, \'SYSTEM DROP MARK CACHE\' = 88, \'SYSTEM DROP UNCOMPRESSED CACHE\' = 89, \'SYSTEM DROP MMAP CACHE\' = 90, \'SYSTEM DROP COMPILED EXPRESSION CACHE\' = 91, \'SYSTEM DROP CACHE\' = 92, \'SYSTEM RELOAD CONFIG\' = 93, \'SYSTEM RELOAD SYMBOLS\' = 94, \'SYSTEM RELOAD DICTIONARY\' = 95, \'SYSTEM RELOAD MODEL\' = 96, \'SYSTEM RELOAD FUNCTION\' = 97, \'SYSTEM RELOAD EMBEDDED DICTIONARIES\' = 98, \'SYSTEM RELOAD\' = 99, \'SYSTEM RESTART DISK\' = 100, \'SYSTEM MERGES\' = 101, \'SYSTEM TTL MERGES\' = 102, \'SYSTEM FETCHES\' = 103, \'SYSTEM MOVES\' = 104, \'SYSTEM DISTRIBUTED SENDS\' = 105, \'SYSTEM REPLICATED SENDS\' = 106, \'SYSTEM SENDS\' = 107, \'SYSTEM REPLICATION QUEUES\' = 108, \'SYSTEM DROP REPLICA\' = 109, \'SYSTEM SYNC REPLICA\' = 110, \'SYSTEM RESTART REPLICA\' = 111, \'SYSTEM RESTORE REPLICA\' = 112, \'SYSTEM FLUSH DISTRIBUTED\' = 113, \'SYSTEM FLUSH LOGS\' = 114, \'SYSTEM FLUSH\' = 115, \'SYSTEM THREAD FUZZER\' = 116, \'SYSTEM\' = 117, \'dictGet\' = 118, \'addressToLine\' = 119, \'addressToSymbol\' = 120, \'demangle\' = 121, \'INTROSPECTION\' = 122, \'FILE\' = 123, \'URL\' = 124, \'REMOTE\' = 125, \'MONGO\' = 126, \'MYSQL\' = 127))\n)\nENGINE = SystemPrivileges()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' +CREATE TABLE system.privileges\n(\n `privilege` Enum16(\'SHOW DATABASES\' = 0, \'SHOW TABLES\' = 1, \'SHOW COLUMNS\' = 2, \'SHOW DICTIONARIES\' = 3, \'SHOW\' = 4, \'SELECT\' = 5, \'INSERT\' = 6, \'ALTER UPDATE\' = 7, \'ALTER DELETE\' = 8, \'ALTER ADD COLUMN\' = 9, \'ALTER MODIFY COLUMN\' = 10, \'ALTER DROP COLUMN\' = 11, \'ALTER COMMENT COLUMN\' = 12, \'ALTER CLEAR COLUMN\' = 13, \'ALTER RENAME COLUMN\' = 14, \'ALTER MATERIALIZE COLUMN\' = 15, \'ALTER COLUMN\' = 16, \'ALTER MODIFY COMMENT\' = 17, \'ALTER ORDER BY\' = 18, \'ALTER SAMPLE BY\' = 19, \'ALTER ADD INDEX\' = 20, \'ALTER DROP INDEX\' = 21, \'ALTER MATERIALIZE INDEX\' = 22, \'ALTER CLEAR INDEX\' = 23, \'ALTER INDEX\' = 24, \'ALTER ADD PROJECTION\' = 25, \'ALTER DROP PROJECTION\' = 26, \'ALTER MATERIALIZE PROJECTION\' = 27, \'ALTER CLEAR PROJECTION\' = 28, \'ALTER PROJECTION\' = 29, \'ALTER ADD CONSTRAINT\' = 30, \'ALTER DROP CONSTRAINT\' = 31, \'ALTER CONSTRAINT\' = 32, \'ALTER TTL\' = 33, \'ALTER MATERIALIZE TTL\' = 34, \'ALTER SETTINGS\' = 35, \'ALTER MOVE PARTITION\' = 36, \'ALTER FETCH PARTITION\' = 37, \'ALTER FREEZE PARTITION\' = 38, \'ALTER DATABASE SETTINGS\' = 39, \'ALTER TABLE\' = 40, \'ALTER DATABASE\' = 41, \'ALTER VIEW REFRESH\' = 42, \'ALTER VIEW MODIFY QUERY\' = 43, \'ALTER VIEW\' = 44, \'ALTER\' = 45, \'CREATE DATABASE\' = 46, \'CREATE TABLE\' = 47, \'CREATE VIEW\' = 48, \'CREATE DICTIONARY\' = 49, \'CREATE TEMPORARY TABLE\' = 50, \'CREATE FUNCTION\' = 51, \'CREATE\' = 52, \'DROP DATABASE\' = 53, \'DROP TABLE\' = 54, \'DROP VIEW\' = 55, \'DROP DICTIONARY\' = 56, \'DROP FUNCTION\' = 57, \'DROP\' = 58, \'TRUNCATE\' = 59, \'OPTIMIZE\' = 60, \'KILL QUERY\' = 61, \'MOVE PARTITION BETWEEN SHARDS\' = 62, \'CREATE USER\' = 63, \'ALTER USER\' = 64, \'DROP USER\' = 65, \'CREATE ROLE\' = 66, \'ALTER ROLE\' = 67, \'DROP ROLE\' = 68, \'ROLE ADMIN\' = 69, \'CREATE ROW POLICY\' = 70, \'ALTER ROW POLICY\' = 71, \'DROP ROW POLICY\' = 72, \'CREATE QUOTA\' = 73, \'ALTER QUOTA\' = 74, \'DROP QUOTA\' = 75, \'CREATE SETTINGS PROFILE\' = 76, \'ALTER SETTINGS PROFILE\' = 77, \'DROP SETTINGS PROFILE\' = 78, \'SHOW USERS\' = 79, \'SHOW ROLES\' = 80, \'SHOW ROW POLICIES\' = 81, \'SHOW QUOTAS\' = 82, \'SHOW SETTINGS PROFILES\' = 83, \'SHOW ACCESS\' = 84, \'ACCESS MANAGEMENT\' = 85, \'SYSTEM SHUTDOWN\' = 86, \'SYSTEM DROP DNS CACHE\' = 87, \'SYSTEM DROP MARK CACHE\' = 88, \'SYSTEM DROP UNCOMPRESSED CACHE\' = 89, \'SYSTEM DROP MMAP CACHE\' = 90, \'SYSTEM DROP COMPILED EXPRESSION CACHE\' = 91, \'SYSTEM DROP CACHE\' = 92, \'SYSTEM RELOAD CONFIG\' = 93, \'SYSTEM RELOAD SYMBOLS\' = 94, \'SYSTEM RELOAD DICTIONARY\' = 95, \'SYSTEM RELOAD MODEL\' = 96, \'SYSTEM RELOAD FUNCTION\' = 97, \'SYSTEM RELOAD EMBEDDED DICTIONARIES\' = 98, \'SYSTEM RELOAD\' = 99, \'SYSTEM RESTART DISK\' = 100, \'SYSTEM MERGES\' = 101, \'SYSTEM TTL MERGES\' = 102, \'SYSTEM FETCHES\' = 103, \'SYSTEM MOVES\' = 104, \'SYSTEM DISTRIBUTED SENDS\' = 105, \'SYSTEM REPLICATED SENDS\' = 106, \'SYSTEM SENDS\' = 107, \'SYSTEM REPLICATION QUEUES\' = 108, \'SYSTEM DROP REPLICA\' = 109, \'SYSTEM SYNC REPLICA\' = 110, \'SYSTEM RESTART REPLICA\' = 111, \'SYSTEM RESTORE REPLICA\' = 112, \'SYSTEM FLUSH DISTRIBUTED\' = 113, \'SYSTEM FLUSH LOGS\' = 114, \'SYSTEM FLUSH\' = 115, \'SYSTEM THREAD FUZZER\' = 116, \'SYSTEM\' = 117, \'dictGet\' = 118, \'addressToLine\' = 119, \'addressToLineWithInlines\' = 120, \'addressToSymbol\' = 121, \'demangle\' = 122, \'INTROSPECTION\' = 123, \'FILE\' = 124, \'URL\' = 125, \'REMOTE\' = 126, \'MONGO\' = 127, \'MYSQL\' = 128, \'POSTGRES\' = 129, \'SQLITE\' = 130, \'ODBC\' = 131, \'JDBC\' = 132, \'HDFS\' = 133, \'S3\' = 134, \'SOURCES\' = 135, \'ALL\' = 136, \'NONE\' = 137),\n `aliases` Array(String),\n `level` Nullable(Enum8(\'GLOBAL\' = 0, \'DATABASE\' = 1, \'TABLE\' = 2, \'DICTIONARY\' = 3, \'VIEW\' = 4, \'COLUMN\' = 5)),\n `parent_group` Nullable(Enum16(\'SHOW DATABASES\' = 0, \'SHOW TABLES\' = 1, \'SHOW COLUMNS\' = 2, \'SHOW DICTIONARIES\' = 3, \'SHOW\' = 4, \'SELECT\' = 5, \'INSERT\' = 6, \'ALTER UPDATE\' = 7, \'ALTER DELETE\' = 8, \'ALTER ADD COLUMN\' = 9, \'ALTER MODIFY COLUMN\' = 10, \'ALTER DROP COLUMN\' = 11, \'ALTER COMMENT COLUMN\' = 12, \'ALTER CLEAR COLUMN\' = 13, \'ALTER RENAME COLUMN\' = 14, \'ALTER MATERIALIZE COLUMN\' = 15, \'ALTER COLUMN\' = 16, \'ALTER MODIFY COMMENT\' = 17, \'ALTER ORDER BY\' = 18, \'ALTER SAMPLE BY\' = 19, \'ALTER ADD INDEX\' = 20, \'ALTER DROP INDEX\' = 21, \'ALTER MATERIALIZE INDEX\' = 22, \'ALTER CLEAR INDEX\' = 23, \'ALTER INDEX\' = 24, \'ALTER ADD PROJECTION\' = 25, \'ALTER DROP PROJECTION\' = 26, \'ALTER MATERIALIZE PROJECTION\' = 27, \'ALTER CLEAR PROJECTION\' = 28, \'ALTER PROJECTION\' = 29, \'ALTER ADD CONSTRAINT\' = 30, \'ALTER DROP CONSTRAINT\' = 31, \'ALTER CONSTRAINT\' = 32, \'ALTER TTL\' = 33, \'ALTER MATERIALIZE TTL\' = 34, \'ALTER SETTINGS\' = 35, \'ALTER MOVE PARTITION\' = 36, \'ALTER FETCH PARTITION\' = 37, \'ALTER FREEZE PARTITION\' = 38, \'ALTER DATABASE SETTINGS\' = 39, \'ALTER TABLE\' = 40, \'ALTER DATABASE\' = 41, \'ALTER VIEW REFRESH\' = 42, \'ALTER VIEW MODIFY QUERY\' = 43, \'ALTER VIEW\' = 44, \'ALTER\' = 45, \'CREATE DATABASE\' = 46, \'CREATE TABLE\' = 47, \'CREATE VIEW\' = 48, \'CREATE DICTIONARY\' = 49, \'CREATE TEMPORARY TABLE\' = 50, \'CREATE FUNCTION\' = 51, \'CREATE\' = 52, \'DROP DATABASE\' = 53, \'DROP TABLE\' = 54, \'DROP VIEW\' = 55, \'DROP DICTIONARY\' = 56, \'DROP FUNCTION\' = 57, \'DROP\' = 58, \'TRUNCATE\' = 59, \'OPTIMIZE\' = 60, \'KILL QUERY\' = 61, \'MOVE PARTITION BETWEEN SHARDS\' = 62, \'CREATE USER\' = 63, \'ALTER USER\' = 64, \'DROP USER\' = 65, \'CREATE ROLE\' = 66, \'ALTER ROLE\' = 67, \'DROP ROLE\' = 68, \'ROLE ADMIN\' = 69, \'CREATE ROW POLICY\' = 70, \'ALTER ROW POLICY\' = 71, \'DROP ROW POLICY\' = 72, \'CREATE QUOTA\' = 73, \'ALTER QUOTA\' = 74, \'DROP QUOTA\' = 75, \'CREATE SETTINGS PROFILE\' = 76, \'ALTER SETTINGS PROFILE\' = 77, \'DROP SETTINGS PROFILE\' = 78, \'SHOW USERS\' = 79, \'SHOW ROLES\' = 80, \'SHOW ROW POLICIES\' = 81, \'SHOW QUOTAS\' = 82, \'SHOW SETTINGS PROFILES\' = 83, \'SHOW ACCESS\' = 84, \'ACCESS MANAGEMENT\' = 85, \'SYSTEM SHUTDOWN\' = 86, \'SYSTEM DROP DNS CACHE\' = 87, \'SYSTEM DROP MARK CACHE\' = 88, \'SYSTEM DROP UNCOMPRESSED CACHE\' = 89, \'SYSTEM DROP MMAP CACHE\' = 90, \'SYSTEM DROP COMPILED EXPRESSION CACHE\' = 91, \'SYSTEM DROP CACHE\' = 92, \'SYSTEM RELOAD CONFIG\' = 93, \'SYSTEM RELOAD SYMBOLS\' = 94, \'SYSTEM RELOAD DICTIONARY\' = 95, \'SYSTEM RELOAD MODEL\' = 96, \'SYSTEM RELOAD FUNCTION\' = 97, \'SYSTEM RELOAD EMBEDDED DICTIONARIES\' = 98, \'SYSTEM RELOAD\' = 99, \'SYSTEM RESTART DISK\' = 100, \'SYSTEM MERGES\' = 101, \'SYSTEM TTL MERGES\' = 102, \'SYSTEM FETCHES\' = 103, \'SYSTEM MOVES\' = 104, \'SYSTEM DISTRIBUTED SENDS\' = 105, \'SYSTEM REPLICATED SENDS\' = 106, \'SYSTEM SENDS\' = 107, \'SYSTEM REPLICATION QUEUES\' = 108, \'SYSTEM DROP REPLICA\' = 109, \'SYSTEM SYNC REPLICA\' = 110, \'SYSTEM RESTART REPLICA\' = 111, \'SYSTEM RESTORE REPLICA\' = 112, \'SYSTEM FLUSH DISTRIBUTED\' = 113, \'SYSTEM FLUSH LOGS\' = 114, \'SYSTEM FLUSH\' = 115, \'SYSTEM THREAD FUZZER\' = 116, \'SYSTEM\' = 117, \'dictGet\' = 118, \'addressToLine\' = 119, \'addressToLineWithInlines\' = 120, \'addressToSymbol\' = 121, \'demangle\' = 122, \'INTROSPECTION\' = 123, \'FILE\' = 124, \'URL\' = 125, \'REMOTE\' = 126, \'MONGO\' = 127, \'MYSQL\' = 128, \'POSTGRES\' = 129, \'SQLITE\' = 130, \'ODBC\' = 131, \'JDBC\' = 132, \'HDFS\' = 133, \'S3\' = 134, \'SOURCES\' = 135, \'ALL\' = 136, \'NONE\' = 137))\n)\nENGINE = SystemPrivileges()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.processes\n(\n `is_initial_query` UInt8,\n `user` String,\n `query_id` String,\n `address` IPv6,\n `port` UInt16,\n `initial_user` String,\n `initial_query_id` String,\n `initial_address` IPv6,\n `initial_port` UInt16,\n `interface` UInt8,\n `os_user` String,\n `client_hostname` String,\n `client_name` String,\n `client_revision` UInt64,\n `client_version_major` UInt64,\n `client_version_minor` UInt64,\n `client_version_patch` UInt64,\n `http_method` UInt8,\n `http_user_agent` String,\n `http_referer` String,\n `forwarded_for` String,\n `quota_key` String,\n `elapsed` Float64,\n `is_cancelled` UInt8,\n `read_rows` UInt64,\n `read_bytes` UInt64,\n `total_rows_approx` UInt64,\n `written_rows` UInt64,\n `written_bytes` UInt64,\n `memory_usage` Int64,\n `peak_memory_usage` Int64,\n `query` String,\n `thread_ids` Array(UInt64),\n `ProfileEvents` Map(String, UInt64),\n `Settings` Map(String, String),\n `current_database` String,\n `ProfileEvents.Names` Array(String),\n `ProfileEvents.Values` Array(UInt64),\n `Settings.Names` Array(String),\n `Settings.Values` Array(String)\n)\nENGINE = SystemProcesses()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.projection_parts\n(\n `partition` String,\n `name` String,\n `part_type` String,\n `parent_name` String,\n `parent_uuid` UUID,\n `parent_part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `parent_marks` UInt64,\n `parent_rows` UInt64,\n `parent_bytes_on_disk` UInt64,\n `parent_data_compressed_bytes` UInt64,\n `parent_data_uncompressed_bytes` UInt64,\n `parent_marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `is_frozen` UInt8,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `hash_of_all_files` String,\n `hash_of_uncompressed_files` String,\n `uncompressed_hash_of_compressed_files` String,\n `delete_ttl_info_min` DateTime,\n `delete_ttl_info_max` DateTime,\n `move_ttl_info.expression` Array(String),\n `move_ttl_info.min` Array(DateTime),\n `move_ttl_info.max` Array(DateTime),\n `default_compression_codec` String,\n `recompression_ttl_info.expression` Array(String),\n `recompression_ttl_info.min` Array(DateTime),\n `recompression_ttl_info.max` Array(DateTime),\n `group_by_ttl_info.expression` Array(String),\n `group_by_ttl_info.min` Array(DateTime),\n `group_by_ttl_info.max` Array(DateTime),\n `rows_where_ttl_info.expression` Array(String),\n `rows_where_ttl_info.min` Array(DateTime),\n `rows_where_ttl_info.max` Array(DateTime),\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemProjectionParts()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' CREATE TABLE system.projection_parts_columns\n(\n `partition` String,\n `name` String,\n `part_type` String,\n `parent_name` String,\n `parent_uuid` UUID,\n `parent_part_type` String,\n `active` UInt8,\n `marks` UInt64,\n `rows` UInt64,\n `bytes_on_disk` UInt64,\n `data_compressed_bytes` UInt64,\n `data_uncompressed_bytes` UInt64,\n `marks_bytes` UInt64,\n `parent_marks` UInt64,\n `parent_rows` UInt64,\n `parent_bytes_on_disk` UInt64,\n `parent_data_compressed_bytes` UInt64,\n `parent_data_uncompressed_bytes` UInt64,\n `parent_marks_bytes` UInt64,\n `modification_time` DateTime,\n `remove_time` DateTime,\n `refcount` UInt32,\n `min_date` Date,\n `max_date` Date,\n `min_time` DateTime,\n `max_time` DateTime,\n `partition_id` String,\n `min_block_number` Int64,\n `max_block_number` Int64,\n `level` UInt32,\n `data_version` UInt64,\n `primary_key_bytes_in_memory` UInt64,\n `primary_key_bytes_in_memory_allocated` UInt64,\n `database` String,\n `table` String,\n `engine` String,\n `disk_name` String,\n `path` String,\n `column` String,\n `type` String,\n `column_position` UInt64,\n `default_kind` String,\n `default_expression` String,\n `column_bytes_on_disk` UInt64,\n `column_data_compressed_bytes` UInt64,\n `column_data_uncompressed_bytes` UInt64,\n `column_marks_bytes` UInt64,\n `bytes` UInt64,\n `marks_size` UInt64\n)\nENGINE = SystemProjectionPartsColumns()\nCOMMENT \'SYSTEM TABLE is built on the fly.\' diff --git a/tests/queries/0_stateless/02119_sumcount.reference b/tests/queries/0_stateless/02119_sumcount.reference index 4adda0e10be..437c52c899a 100644 --- a/tests/queries/0_stateless/02119_sumcount.reference +++ b/tests/queries/0_stateless/02119_sumcount.reference @@ -2,10 +2,10 @@ Tuple(UInt64, UInt64) (9007199254740994,3) Tuple(UInt64, UInt64) (9007199254740994,3) Tuple(UInt64, UInt64) (9007199254740994,3) Tuple(UInt64, UInt64) (9007199254740994,3) -Tuple(Float64, UInt64) (9007199254740992,3) -Tuple(Float64, UInt64) (9007199254740992,3) -Tuple(Float64, UInt64) (9007199254740992,3) -Tuple(Float64, UInt64) (9007199254740992,3) +Tuple(Float64, UInt64) (9007199254740994,3) +Tuple(Float64, UInt64) (9007199254740994,3) +Tuple(Float64, UInt64) (9007199254740994,3) +Tuple(Float64, UInt64) (9007199254740994,3) Tuple(Float64, UInt64) (16777218,3) Tuple(Float64, UInt64) (16777218,3) Tuple(Float64, UInt64) (16777218,3) diff --git a/tests/queries/0_stateless/02119_sumcount.sql b/tests/queries/0_stateless/02119_sumcount.sql index 9189efd08ac..22cb8b657da 100644 --- a/tests/queries/0_stateless/02119_sumcount.sql +++ b/tests/queries/0_stateless/02119_sumcount.sql @@ -1,20 +1,150 @@ -- Integer types are added as integers -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '9007199254740992'::UInt64 AS v UNION ALL SELECT '1'::UInt64 AS v UNION ALL SELECT '1'::UInt64 AS v); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '9007199254740992'::Nullable(UInt64) AS v UNION ALL SELECT '1'::Nullable(UInt64) AS v UNION ALL SELECT '1'::Nullable(UInt64) AS v ); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '9007199254740992'::LowCardinality(UInt64) AS v UNION ALL SELECT '1'::LowCardinality(UInt64) AS v UNION ALL SELECT '1'::LowCardinality(UInt64) AS v ); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '9007199254740992'::LowCardinality(Nullable(UInt64)) AS v UNION ALL SELECT '1'::LowCardinality(Nullable(UInt64)) AS v UNION ALL SELECT '1'::LowCardinality(Nullable(UInt64)) AS v ); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '9007199254740992'::UInt64 AS v + UNION ALL + SELECT '1'::UInt64 AS v + UNION ALL SELECT '1'::UInt64 AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '9007199254740992'::Nullable(UInt64) AS v + UNION ALL + SELECT '1'::Nullable(UInt64) AS v + UNION ALL + SELECT '1'::Nullable(UInt64) AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '9007199254740992'::LowCardinality(UInt64) AS v + UNION ALL + SELECT '1'::LowCardinality(UInt64) AS v + UNION ALL + SELECT '1'::LowCardinality(UInt64) AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '9007199254740992'::LowCardinality(Nullable(UInt64)) AS v + UNION ALL + SELECT '1'::LowCardinality(Nullable(UInt64)) AS v + UNION ALL + SELECT '1'::LowCardinality(Nullable(UInt64)) AS v + ) + ORDER BY v +); --- Float64 types are added as Float64 -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '9007199254740992'::Float64 AS v UNION ALL SELECT '1'::Float64 AS v UNION ALL SELECT '1'::Float64 AS v ); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '9007199254740992'::Nullable(Float64) AS v UNION ALL SELECT '1'::Nullable(Float64) AS v UNION ALL SELECT '1'::Nullable(Float64) AS v ); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '9007199254740992'::LowCardinality(Float64) AS v UNION ALL SELECT '1'::LowCardinality(Float64) AS v UNION ALL SELECT '1'::LowCardinality(Float64) AS v); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '9007199254740992'::LowCardinality(Nullable(Float64)) AS v UNION ALL SELECT '1'::LowCardinality(Nullable(Float64)) AS v UNION ALL SELECT '1'::LowCardinality(Nullable(Float64)) AS v ); +-- -- Float64 types are added as Float64 +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '9007199254740992'::Float64 AS v + UNION ALL + SELECT '1'::Float64 AS v + UNION ALL SELECT '1'::Float64 AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '9007199254740992'::Nullable(Float64) AS v + UNION ALL + SELECT '1'::Nullable(Float64) AS v + UNION ALL + SELECT '1'::Nullable(Float64) AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '9007199254740992'::LowCardinality(Float64) AS v + UNION ALL + SELECT '1'::LowCardinality(Float64) AS v + UNION ALL + SELECT '1'::LowCardinality(Float64) AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '9007199254740992'::LowCardinality(Nullable(Float64)) AS v + UNION ALL + SELECT '1'::LowCardinality(Nullable(Float64)) AS v + UNION ALL + SELECT '1'::LowCardinality(Nullable(Float64)) AS v + ) + ORDER BY v +); --- Float32 are added as Float64 -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '16777216'::Float32 AS v UNION ALL SELECT '1'::Float32 AS v UNION ALL SELECT '1'::Float32 AS v ); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '16777216'::Nullable(Float32) AS v UNION ALL SELECT '1'::Nullable(Float32) AS v UNION ALL SELECT '1'::Nullable(Float32) AS v ); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '16777216'::LowCardinality(Float32) AS v UNION ALL SELECT '1'::LowCardinality(Float32) AS v UNION ALL SELECT '1'::LowCardinality(Float32) AS v ); -SELECT toTypeName(sumCount(v)), sumCount(v) FROM (SELECT '16777216'::LowCardinality(Nullable(Float32)) AS v UNION ALL SELECT '1'::LowCardinality(Nullable(Float32)) AS v UNION ALL SELECT '1'::LowCardinality(Nullable(Float32)) AS v ); +-- -- Float32 are added as Float64 +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '16777216'::Float32 AS v + UNION ALL + SELECT '1'::Float32 AS v + UNION ALL + SELECT '1'::Float32 AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '16777216'::Nullable(Float32) AS v + UNION ALL + SELECT '1'::Nullable(Float32) AS v + UNION ALL + SELECT '1'::Nullable(Float32) AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '16777216'::LowCardinality(Float32) AS v + UNION ALL + SELECT '1'::LowCardinality(Float32) AS v + UNION ALL + SELECT '1'::LowCardinality(Float32) AS v + ) + ORDER BY v +); +SELECT toTypeName(sumCount(v)), sumCount(v) FROM +( + SELECT v FROM + ( + SELECT '16777216'::LowCardinality(Nullable(Float32)) AS v + UNION ALL + SELECT '1'::LowCardinality(Nullable(Float32)) AS v + UNION ALL + SELECT '1'::LowCardinality(Nullable(Float32)) AS v + ) + ORDER BY v +); -- Small integer types use their sign/unsigned 64 byte supertype SELECT toTypeName(sumCount(number::Int8)), sumCount(number::Int8) FROM numbers(120); diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token.reference b/tests/queries/0_stateless/02124_insert_deduplication_token.reference new file mode 100644 index 00000000000..a29d74fdcd1 --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token.reference @@ -0,0 +1,15 @@ +create and check deduplication +two inserts with exact data, one inserted, one deduplicated by data digest +0 1000 +two inserts with the same dedup token, one inserted, one deduplicated by the token +0 1000 +1 1001 +update dedup token, two inserts with the same dedup token, one inserted, one deduplicated by the token +0 1000 +1 1001 +1 1001 +reset deduplication token and insert new row +0 1000 +1 1001 +1 1001 +2 1002 diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token.sql b/tests/queries/0_stateless/02124_insert_deduplication_token.sql new file mode 100644 index 00000000000..4581ef995fd --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token.sql @@ -0,0 +1,33 @@ +-- insert data duplicates by providing deduplication token on insert + +DROP TABLE IF EXISTS insert_dedup_token SYNC; + +select 'create and check deduplication'; +CREATE TABLE insert_dedup_token ( + id Int32, val UInt32 +) ENGINE=MergeTree() ORDER BY id +SETTINGS non_replicated_deduplication_window=0xFFFFFFFF; + +select 'two inserts with exact data, one inserted, one deduplicated by data digest'; +INSERT INTO insert_dedup_token VALUES(0, 1000); +INSERT INTO insert_dedup_token VALUES(0, 1000); +SELECT * FROM insert_dedup_token ORDER BY id; + +select 'two inserts with the same dedup token, one inserted, one deduplicated by the token'; +set insert_deduplication_token = '\x61\x00\x62'; +INSERT INTO insert_dedup_token VALUES(1, 1001); +INSERT INTO insert_dedup_token VALUES(2, 1002); +SELECT * FROM insert_dedup_token ORDER BY id; + +select 'update dedup token, two inserts with the same dedup token, one inserted, one deduplicated by the token'; +set insert_deduplication_token = '\x61\x00\x63'; +INSERT INTO insert_dedup_token VALUES(1, 1001); +INSERT INTO insert_dedup_token VALUES(2, 1002); +SELECT * FROM insert_dedup_token ORDER BY id; + +select 'reset deduplication token and insert new row'; +set insert_deduplication_token = ''; +INSERT INTO insert_dedup_token VALUES(2, 1002); +SELECT * FROM insert_dedup_token ORDER BY id; + +DROP TABLE insert_dedup_token SYNC; diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.reference b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.reference new file mode 100644 index 00000000000..5cf6230fd85 --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.reference @@ -0,0 +1,34 @@ +insert 2 blocks with dedup token, 1 row per block +2 +1 +2 +insert deduplicated by token +2 +1 +2 +insert the same data by providing different dedup token +4 +1 +1 +2 +2 +insert 4 blocks, 2 deduplicated, 2 inserted +6 +1 +1 +2 +2 +3 +4 +disable token based deduplication, insert the same data as with token +10 +1 +1 +1 +2 +2 +2 +3 +3 +4 +4 diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh new file mode 100755 index 00000000000..04ccbda6235 --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +QUERY_COUNT_ORIGIN_BLOCKS="SELECT COUNT(*) FROM system.parts WHERE database = currentDatabase() AND table = 'block_dedup_token' AND min_block_number == max_block_number;" +QUERY_SELECT_FROM_TABLE_ORDERED="SELECT * FROM block_dedup_token ORDER BY id;" +INSERT_BLOCK_SETTINGS="max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS block_dedup_token SYNC" +$CLICKHOUSE_CLIENT --query="CREATE TABLE block_dedup_token (id Int32) ENGINE=MergeTree() ORDER BY id SETTINGS non_replicated_deduplication_window=0xFFFFFFFF;" + +$CLICKHOUSE_CLIENT --query="SELECT 'insert 2 blocks with dedup token, 1 row per block'" +DEDUP_TOKEN='dedup1' +echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="SELECT 'insert deduplicated by token'" +echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="SELECT 'insert the same data by providing different dedup token'" +DEDUP_TOKEN='dedup2' +echo 'INSERT INTO block_dedup_token VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="SELECT 'insert 4 blocks, 2 deduplicated, 2 inserted'" +echo 'INSERT INTO block_dedup_token VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="SELECT 'disable token based deduplication, insert the same data as with token'" +DEDUP_TOKEN='' +echo 'INSERT INTO block_dedup_token VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="DROP TABLE block_dedup_token SYNC" diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.reference b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.reference new file mode 100644 index 00000000000..5cf6230fd85 --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.reference @@ -0,0 +1,34 @@ +insert 2 blocks with dedup token, 1 row per block +2 +1 +2 +insert deduplicated by token +2 +1 +2 +insert the same data by providing different dedup token +4 +1 +1 +2 +2 +insert 4 blocks, 2 deduplicated, 2 inserted +6 +1 +1 +2 +2 +3 +4 +disable token based deduplication, insert the same data as with token +10 +1 +1 +1 +2 +2 +2 +3 +3 +4 +4 diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh new file mode 100755 index 00000000000..1c776263f78 --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_multiple_blocks_replica.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +QUERY_COUNT_ORIGIN_BLOCKS="SELECT COUNT(*) FROM system.parts WHERE database = currentDatabase() AND table = 'block_dedup_token_replica' AND min_block_number == max_block_number;" +QUERY_SELECT_FROM_TABLE_ORDERED="SELECT * FROM block_dedup_token_replica ORDER BY id;" +INSERT_BLOCK_SETTINGS="max_insert_block_size=1&min_insert_block_size_rows=0&min_insert_block_size_bytes=0" + +$CLICKHOUSE_CLIENT --query="DROP TABLE IF EXISTS block_dedup_token_replica SYNC" +$CLICKHOUSE_CLIENT --query="CREATE TABLE block_dedup_token_replica (id Int32) ENGINE=ReplicatedMergeTree('/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/{table}', '{replica}') ORDER BY id" + +$CLICKHOUSE_CLIENT --query="SELECT 'insert 2 blocks with dedup token, 1 row per block'" +DEDUP_TOKEN='dedup1' +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="SELECT 'insert deduplicated by token'" +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="SELECT 'insert the same data by providing different dedup token'" +DEDUP_TOKEN='dedup2' +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="SELECT 'insert 4 blocks, 2 deduplicated, 2 inserted'" +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="SELECT 'disable token based deduplication, insert the same data as with token'" +DEDUP_TOKEN='' +echo 'INSERT INTO block_dedup_token_replica VALUES (1), (2), (3), (4)' | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&$INSERT_BLOCK_SETTINGS&insert_deduplication_token='$DEDUP_TOKEN'&query=" --data-binary @- +$CLICKHOUSE_CLIENT --multiquery --query "$QUERY_COUNT_ORIGIN_BLOCKS;$QUERY_SELECT_FROM_TABLE_ORDERED" + +$CLICKHOUSE_CLIENT --query="DROP TABLE block_dedup_token_replica SYNC" diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_replica.reference b/tests/queries/0_stateless/02124_insert_deduplication_token_replica.reference new file mode 100644 index 00000000000..27691557d46 --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_replica.reference @@ -0,0 +1,24 @@ +create replica 1 and check deduplication +two inserts with exact data, one inserted, one deduplicated by data digest +1 1001 +two inserts with the same dedup token, one inserted, one deduplicated by the token +1 1001 +1 1001 +reset deduplication token and insert new row +1 1001 +1 1001 +2 1002 +create replica 2 and check deduplication +inserted value deduplicated by data digest, the same result as before +1 1001 +1 1001 +2 1002 +inserted value deduplicated by dedup token, the same result as before +1 1001 +1 1001 +2 1002 +new record inserted by providing new deduplication token +1 1001 +1 1001 +2 1002 +2 1002 diff --git a/tests/queries/0_stateless/02124_insert_deduplication_token_replica.sql b/tests/queries/0_stateless/02124_insert_deduplication_token_replica.sql new file mode 100644 index 00000000000..47f33a4a971 --- /dev/null +++ b/tests/queries/0_stateless/02124_insert_deduplication_token_replica.sql @@ -0,0 +1,49 @@ +-- insert data duplicates by providing deduplication token on insert + +DROP TABLE IF EXISTS insert_dedup_token1 SYNC; +DROP TABLE IF EXISTS insert_dedup_token2 SYNC; + +select 'create replica 1 and check deduplication'; +CREATE TABLE insert_dedup_token1 ( + id Int32, val UInt32 +) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/insert_dedup_token', 'r1') ORDER BY id; + +select 'two inserts with exact data, one inserted, one deduplicated by data digest'; +INSERT INTO insert_dedup_token1 VALUES(1, 1001); +INSERT INTO insert_dedup_token1 VALUES(1, 1001); +SELECT * FROM insert_dedup_token1 ORDER BY id; + +select 'two inserts with the same dedup token, one inserted, one deduplicated by the token'; +set insert_deduplication_token = '1'; +INSERT INTO insert_dedup_token1 VALUES(1, 1001); +INSERT INTO insert_dedup_token1 VALUES(2, 1002); +SELECT * FROM insert_dedup_token1 ORDER BY id; + +select 'reset deduplication token and insert new row'; +set insert_deduplication_token = ''; +INSERT INTO insert_dedup_token1 VALUES(2, 1002); +SELECT * FROM insert_dedup_token1 ORDER BY id; + +select 'create replica 2 and check deduplication'; +CREATE TABLE insert_dedup_token2 ( + id Int32, val UInt32 +) ENGINE=ReplicatedMergeTree('/clickhouse/tables/{database}/insert_dedup_token', 'r2') ORDER BY id; +SYSTEM SYNC REPLICA insert_dedup_token2; + +select 'inserted value deduplicated by data digest, the same result as before'; +set insert_deduplication_token = ''; +INSERT INTO insert_dedup_token2 VALUES(1, 1001); -- deduplicated by data digest +SELECT * FROM insert_dedup_token2 ORDER BY id; + +select 'inserted value deduplicated by dedup token, the same result as before'; +set insert_deduplication_token = '1'; +INSERT INTO insert_dedup_token2 VALUES(3, 1003); -- deduplicated by dedup token +SELECT * FROM insert_dedup_token2 ORDER BY id; + +select 'new record inserted by providing new deduplication token'; +set insert_deduplication_token = '2'; +INSERT INTO insert_dedup_token2 VALUES(2, 1002); -- inserted +SELECT * FROM insert_dedup_token2 ORDER BY id; + +DROP TABLE insert_dedup_token1 SYNC; +DROP TABLE insert_dedup_token2 SYNC; diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference new file mode 100644 index 00000000000..9e24b7c6ea6 --- /dev/null +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.reference @@ -0,0 +1,155 @@ +2020-10-01 0 +2020-10-01 0 +2020-10-01 0 +2020-10-01 0 +2020-10-01 0 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder × 2 0 → 1 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (SettingQuotaAndLimits) + (ReadFromMergeTree) + ReverseTransform + MergeTreeReverse 0 → 1 + ReverseTransform + MergeTreeReverse 0 → 1 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +2020-10-01 9 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + FinishSortingTransform + PartialSortingTransform + MergingSortedTransform 2 → 1 + (Expression) + ExpressionTransform × 2 + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder × 2 0 → 1 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +2020-10-11 0 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (Filter) + FilterTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +2020-10-11 0 0 +2020-10-11 0 10 +2020-10-11 0 20 +2020-10-11 0 30 +2020-10-11 0 40 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + FinishSortingTransform + PartialSortingTransform + (Expression) + ExpressionTransform + (Filter) + FilterTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +2020-10-12 0 +2020-10-12 1 +2020-10-12 2 +2020-10-12 3 +2020-10-12 4 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (Filter) + FilterTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + ReverseTransform + MergeTreeReverse 0 → 1 +2020-10-12 99999 +2020-10-12 99998 +2020-10-12 99997 +2020-10-12 99996 +2020-10-12 99995 +1 2 +1 2 +1 3 +1 3 +1 4 +1 4 +======== +1 4 +1 4 +1 3 +1 3 +1 2 +1 2 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +(Expression) +ExpressionTransform + (Limit) + Limit + (Sorting) + (Expression) + ExpressionTransform + (SettingQuotaAndLimits) + (ReadFromMergeTree) + MergeTreeInOrder 0 → 1 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 +2020-10-10 00:00:00 0.01 diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql new file mode 100644 index 00000000000..7d0e9111d9c --- /dev/null +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix.sql @@ -0,0 +1,58 @@ +DROP TABLE IF EXISTS t_read_in_order; + +CREATE TABLE t_read_in_order(date Date, i UInt64, v UInt64) +ENGINE = MergeTree ORDER BY (date, i); + +INSERT INTO t_read_in_order SELECT '2020-10-10', number % 10, number FROM numbers(100000); +INSERT INTO t_read_in_order SELECT '2020-10-11', number % 10, number FROM numbers(100000); + +SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, i LIMIT 5; +EXPLAIN PIPELINE SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, i LIMIT 5; + +SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d DESC, -i LIMIT 5; +EXPLAIN PIPELINE SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d DESC, -i LIMIT 5; + +-- Here FinishSorting is used, because directions don't match. +SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, -i LIMIT 5; +EXPLAIN PIPELINE SELECT toStartOfMonth(date) as d, i FROM t_read_in_order ORDER BY d, -i LIMIT 5; + +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5; +EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i LIMIT 5; + +SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5; +EXPLAIN PIPELINE SELECT * FROM t_read_in_order WHERE date = '2020-10-11' ORDER BY i, v LIMIT 5; + +INSERT INTO t_read_in_order SELECT '2020-10-12', number, number FROM numbers(100000); + +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i LIMIT 5; + +EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5; +SELECT date, i FROM t_read_in_order WHERE date = '2020-10-12' ORDER BY i DESC LIMIT 5; + +DROP TABLE IF EXISTS t_read_in_order; + +CREATE TABLE t_read_in_order(a UInt32, b UInt32) +ENGINE = MergeTree ORDER BY (a, b) +SETTINGS index_granularity = 3; + +SYSTEM STOP MERGES t_read_in_order; + +INSERT INTO t_read_in_order VALUES (0, 100), (1, 2), (1, 3), (1, 4), (2, 5); +INSERT INTO t_read_in_order VALUES (0, 100), (1, 2), (1, 3), (1, 4), (2, 5); + +SELECT a, b FROM t_read_in_order WHERE a = 1 ORDER BY b SETTINGS read_in_order_two_level_merge_threshold = 1; +SELECT '========'; +SELECT a, b FROM t_read_in_order WHERE a = 1 ORDER BY b DESC SETTINGS read_in_order_two_level_merge_threshold = 1; + +DROP TABLE t_read_in_order; + +CREATE TABLE t_read_in_order(dt DateTime, d Decimal64(5), v UInt64) +ENGINE = MergeTree ORDER BY (toStartOfDay(dt), d); + +INSERT INTO t_read_in_order SELECT toDateTime('2020-10-10 00:00:00') + number, 1 / (number % 100 + 1), number FROM numbers(1000); + +EXPLAIN PIPELINE SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; +SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; + +EXPLAIN PIPELINE SELECT toStartOfDay(dt) as date, d FROM t_read_in_order ORDER BY date, round(d) LIMIT 5; +SELECT toStartOfDay(dt) as date, d FROM t_read_in_order WHERE date = '2020-10-10' ORDER BY round(d) LIMIT 5; diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.reference b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.reference new file mode 100644 index 00000000000..4a3cbec8ce3 --- /dev/null +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.reference @@ -0,0 +1,5 @@ +MergeSorting +MergeSorting +MergeSorting +MergeSorting +MergeSorting diff --git a/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.sh b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.sh new file mode 100755 index 00000000000..a2f297da5ad --- /dev/null +++ b/tests/queries/0_stateless/02149_read_in_order_fixed_prefix_negative.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_read_in_order_2"; + +$CLICKHOUSE_CLIENT -q "CREATE TABLE t_read_in_order_2(date Date, i UInt64, v UInt64) ENGINE = MergeTree ORDER BY (date, i)" + +$CLICKHOUSE_CLIENT -q "INSERT INTO t_read_in_order_2 SELECT '2020-10-10', number % 10, number FROM numbers(100000)" +$CLICKHOUSE_CLIENT -q "INSERT INTO t_read_in_order_2 SELECT '2020-10-11', number % 10, number FROM numbers(100000)" + +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE date = '2020-10-11' OR date = '2020-10-12' ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE date >= '2020-10-11' ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE date = '2020-10-11' OR v = 100 ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE date != '2020-10-11' ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" +$CLICKHOUSE_CLIENT -q "EXPLAIN PIPELINE SELECT date, i FROM t_read_in_order_2 WHERE NOT (date = '2020-10-11') ORDER BY i DESC LIMIT 10" | grep -o "MergeSorting" + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS t_read_in_order_2"; diff --git a/tests/queries/0_stateless/02154_dictionary_get_http_json.reference b/tests/queries/0_stateless/02154_dictionary_get_http_json.reference new file mode 100644 index 00000000000..7106f551cd7 --- /dev/null +++ b/tests/queries/0_stateless/02154_dictionary_get_http_json.reference @@ -0,0 +1,24 @@ +0 Value +{ + "meta": + [ + { + "name": "dictGet(02154_test_dictionary, 'value', toUInt64(0))", + "type": "String" + }, + { + "name": "dictGet(02154_test_dictionary, 'value', toUInt64(1))", + "type": "String" + } + ], + + "data": + [ + { + "dictGet(02154_test_dictionary, 'value', toUInt64(0))": "Value", + "dictGet(02154_test_dictionary, 'value', toUInt64(1))": "" + } + ], + + "rows": 1 +} diff --git a/tests/queries/0_stateless/02154_dictionary_get_http_json.sh b/tests/queries/0_stateless/02154_dictionary_get_http_json.sh new file mode 100755 index 00000000000..a2bce866c76 --- /dev/null +++ b/tests/queries/0_stateless/02154_dictionary_get_http_json.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "DROP TABLE IF EXISTS 02154_test_source_table" + +$CLICKHOUSE_CLIENT -q """ + CREATE TABLE 02154_test_source_table + ( + id UInt64, + value String + ) ENGINE=TinyLog; +""" + +$CLICKHOUSE_CLIENT -q "INSERT INTO 02154_test_source_table VALUES (0, 'Value')" +$CLICKHOUSE_CLIENT -q "SELECT * FROM 02154_test_source_table" + +$CLICKHOUSE_CLIENT -q "DROP DICTIONARY IF EXISTS 02154_test_dictionary" +$CLICKHOUSE_CLIENT -q """ + CREATE DICTIONARY 02154_test_dictionary + ( + id UInt64, + value String + ) + PRIMARY KEY id + LAYOUT(HASHED()) + LIFETIME(0) + SOURCE(CLICKHOUSE(TABLE '02154_test_source_table')) +""" + +echo """ + SELECT dictGet(02154_test_dictionary, 'value', toUInt64(0)), dictGet(02154_test_dictionary, 'value', toUInt64(1)) + FORMAT JSON +""" | ${CLICKHOUSE_CURL} -sSg "${CLICKHOUSE_URL}&wait_end_of_query=1&output_format_write_statistics=0" -d @- + +$CLICKHOUSE_CLIENT -q "DROP DICTIONARY 02154_test_dictionary" +$CLICKHOUSE_CLIENT -q "DROP TABLE 02154_test_source_table" diff --git a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect new file mode 100755 index 00000000000..244e48226e5 --- /dev/null +++ b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.expect @@ -0,0 +1,86 @@ +#!/usr/bin/expect -f + +log_user 0 +set timeout 60 +set uuid "" +match_max 100000 +expect_after { + # Do not ignore eof from read. + eof { exp_continue } + # A default timeout action is to do nothing, change it to fail + timeout { exit 1 } +} + +set basedir [file dirname $argv0] +spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_CLIENT_BINARY \$CLICKHOUSE_CLIENT_OPT" +expect ":) " + +# Make a query +send -- "set max_distributed" +expect "set max_distributed" + +# Wait for suggestions to load, they are loaded in background +set is_done 0 +set timeout 1 +while {$is_done == 0} { + send -- "\t" + expect { + "_" { + set is_done 1 + } + default { + # Reset the expect_after + } + } +} +set timeout 60 +# Ctrl-C +send -- "\3" +expect ":) " + +# Generate UIUD to avoid matching old database/tables/columns from previous test runs. +send -- "select 'begin-' || replace(toString(generateUUIDv4()), '-', '') || '-end' format TSV\r" +expect -re TSV.*TSV.*begin-(.*)-end.* +set uuid $expect_out(1,string) +expect ":) " + +# Create +send -- "create database new_${uuid}_database\r" +expect ":) " +send -- "create table new_${uuid}_table (new_${uuid}_column Int) engine=Null()\r" +expect ":) " + +# Check completion +send -- "new_${uuid}_data" +expect "new_${uuid}_data" +send -- "\t" +expect "base" +# Ctrl-C +send -- "\3" +expect ":) " + +send -- "new_${uuid}_ta" +expect "new_${uuid}_ta" +send -- "\t" +expect "ble" +# Ctrl-C +send -- "\3" +expect ":) " + +send -- "new_${uuid}_col" +expect "new_${uuid}_col" +send -- "\t" +expect "umn" +# Ctrl-C +send -- "\3" +expect ":) " + +# Cleanup +send -- "drop database new_${uuid}_database\r" +expect ":) " +send -- "drop table new_${uuid}_table\r" +expect ":) " + +# Ctrl-D +send -- "\4" +expect eof diff --git a/tests/queries/0_stateless/02160_client_autocomplete_parse_query.reference b/tests/queries/0_stateless/02160_client_autocomplete_parse_query.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02161_addressToLineWithInlines.reference b/tests/queries/0_stateless/02161_addressToLineWithInlines.reference new file mode 100644 index 00000000000..10e2c7069b3 --- /dev/null +++ b/tests/queries/0_stateless/02161_addressToLineWithInlines.reference @@ -0,0 +1,2 @@ +10000000000 +has inlines: 1 diff --git a/tests/queries/0_stateless/02161_addressToLineWithInlines.sql b/tests/queries/0_stateless/02161_addressToLineWithInlines.sql new file mode 100644 index 00000000000..baddea30ae3 --- /dev/null +++ b/tests/queries/0_stateless/02161_addressToLineWithInlines.sql @@ -0,0 +1,25 @@ +-- Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug + + +SELECT addressToLineWithInlines(1); -- { serverError 446 } + +SET allow_introspection_functions = 1; +SET query_profiler_real_time_period_ns = 0; +SET query_profiler_cpu_time_period_ns = 1000000; +SET log_queries = 1; +SELECT count() FROM numbers_mt(10000000000) SETTINGS log_comment='02161_test_case'; +SET log_queries = 0; +SET query_profiler_cpu_time_period_ns = 0; +SYSTEM FLUSH LOGS; + +WITH + lineWithInlines AS + ( + SELECT DISTINCT addressToLineWithInlines(arrayJoin(trace)) AS lineWithInlines FROM system.trace_log WHERE query_id = + ( + SELECT query_id FROM system.query_log WHERE current_database = currentDatabase() AND log_comment='02161_test_case' ORDER BY event_time DESC LIMIT 1 + ) + ) +SELECT 'has inlines:', or(max(length(lineWithInlines)) > 1, max(locate(lineWithInlines[1], ':')) = 0) FROM lineWithInlines SETTINGS short_circuit_function_evaluation='enable'; +-- `max(length(lineWithInlines)) > 1` check there is any inlines. +-- `max(locate(lineWithInlines[1], ':')) = 0` check whether none could get a symbol. diff --git a/tests/queries/0_stateless/02163_shard_num.reference b/tests/queries/0_stateless/02163_shard_num.reference new file mode 100644 index 00000000000..a109d5d2b6b --- /dev/null +++ b/tests/queries/0_stateless/02163_shard_num.reference @@ -0,0 +1,17 @@ +-- { echo } +SELECT shardNum() AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY _shard_num; +2 1 +1 1 +SELECT shardNum() AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY shard_num; +2 1 +1 1 +SELECT _shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY _shard_num; +2 1 +1 1 +SELECT _shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY shard_num; +2 1 +1 1 +SELECT a._shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) a GROUP BY shard_num; +2 1 +1 1 +SELECT _shard_num FROM remote('127.1', system.one) AS a INNER JOIN (SELECT _shard_num FROM system.one) AS b USING (dummy); -- { serverError UNKNOWN_IDENTIFIER } diff --git a/tests/queries/0_stateless/02163_shard_num.sql b/tests/queries/0_stateless/02163_shard_num.sql new file mode 100644 index 00000000000..27d40b3c976 --- /dev/null +++ b/tests/queries/0_stateless/02163_shard_num.sql @@ -0,0 +1,7 @@ +-- { echo } +SELECT shardNum() AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY _shard_num; +SELECT shardNum() AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY shard_num; +SELECT _shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY _shard_num; +SELECT _shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) GROUP BY shard_num; +SELECT a._shard_num AS shard_num, sum(1) as rows FROM remote('127.{1,2}', system, one) a GROUP BY shard_num; +SELECT _shard_num FROM remote('127.1', system.one) AS a INNER JOIN (SELECT _shard_num FROM system.one) AS b USING (dummy); -- { serverError UNKNOWN_IDENTIFIER } diff --git a/tests/queries/0_stateless/02174_cte_scalar_cache.reference b/tests/queries/0_stateless/02174_cte_scalar_cache.reference new file mode 100644 index 00000000000..88456b1e7ea --- /dev/null +++ b/tests/queries/0_stateless/02174_cte_scalar_cache.reference @@ -0,0 +1,2 @@ +02177_CTE_GLOBAL_ON 5 500 11 0 5 +02177_CTE_GLOBAL_OFF 1 100 5 0 1 diff --git a/tests/queries/0_stateless/02174_cte_scalar_cache.sql b/tests/queries/0_stateless/02174_cte_scalar_cache.sql new file mode 100644 index 00000000000..4b015cdd007 --- /dev/null +++ b/tests/queries/0_stateless/02174_cte_scalar_cache.sql @@ -0,0 +1,48 @@ +WITH + ( SELECT sleep(0.0001) FROM system.one ) as a1, + ( SELECT sleep(0.0001) FROM system.one ) as a2, + ( SELECT sleep(0.0001) FROM system.one ) as a3, + ( SELECT sleep(0.0001) FROM system.one ) as a4, + ( SELECT sleep(0.0001) FROM system.one ) as a5 +SELECT '02177_CTE_GLOBAL_ON', a5 FROM system.numbers LIMIT 100 +FORMAT Null +SETTINGS enable_global_with_statement = 1; + +WITH + ( SELECT sleep(0.0001) FROM system.one ) as a1, + ( SELECT sleep(0.0001) FROM system.one ) as a2, + ( SELECT sleep(0.0001) FROM system.one ) as a3, + ( SELECT sleep(0.0001) FROM system.one ) as a4, + ( SELECT sleep(0.0001) FROM system.one ) as a5 +SELECT '02177_CTE_GLOBAL_OFF', a5 FROM system.numbers LIMIT 100 + FORMAT Null +SETTINGS enable_global_with_statement = 0; + +SYSTEM FLUSH LOGS; +SELECT + '02177_CTE_GLOBAL_ON', + ProfileEvents['SleepFunctionCalls'] as sleep_calls, + ProfileEvents['SleepFunctionMicroseconds'] as sleep_microseconds, + ProfileEvents['ScalarSubqueriesGlobalCacheHit'] as scalar_cache_global_hit, + ProfileEvents['ScalarSubqueriesLocalCacheHit'] as scalar_cache_local_hit, + ProfileEvents['ScalarSubqueriesCacheMiss'] as scalar_cache_miss +FROM system.query_log +WHERE + current_database = currentDatabase() + AND type = 'QueryFinish' + AND query LIKE '%SELECT ''02177_CTE_GLOBAL_ON%' + AND event_date >= yesterday() AND event_time > now() - interval 10 minute; + +SELECT + '02177_CTE_GLOBAL_OFF', + ProfileEvents['SleepFunctionCalls'] as sleep_calls, + ProfileEvents['SleepFunctionMicroseconds'] as sleep_microseconds, + ProfileEvents['ScalarSubqueriesGlobalCacheHit'] as scalar_cache_global_hit, + ProfileEvents['ScalarSubqueriesLocalCacheHit'] as scalar_cache_local_hit, + ProfileEvents['ScalarSubqueriesCacheMiss'] as scalar_cache_miss +FROM system.query_log +WHERE + current_database = currentDatabase() + AND type = 'QueryFinish' + AND query LIKE '%02177_CTE_GLOBAL_OFF%' + AND event_date >= yesterday() AND event_time > now() - interval 10 minute; diff --git a/tests/queries/0_stateless/02174_cte_scalar_cache_mv.reference b/tests/queries/0_stateless/02174_cte_scalar_cache_mv.reference new file mode 100644 index 00000000000..246706164df --- /dev/null +++ b/tests/queries/0_stateless/02174_cte_scalar_cache_mv.reference @@ -0,0 +1,63 @@ +4 4 4 4 5 +9 9 9 9 5 +14 14 14 14 5 +19 19 19 19 5 +24 24 24 24 5 +29 29 29 29 5 +34 34 34 34 5 +39 39 39 39 5 +44 44 44 44 5 +49 49 49 49 5 +54 54 54 54 5 +59 59 59 59 5 +64 64 64 64 5 +69 69 69 69 5 +74 74 74 74 5 +79 79 79 79 5 +84 84 84 84 5 +89 89 89 89 5 +94 94 94 94 5 +99 99 99 99 5 +02177_MV 7 80 22 +10 +40 +70 +100 +130 +160 +190 +220 +250 +280 +310 +340 +370 +400 +430 +460 +490 +520 +550 +580 +02177_MV_2 0 0 21 +8 +18 +28 +38 +48 +58 +68 +78 +88 +98 +108 +118 +128 +138 +148 +158 +168 +178 +188 +198 +02177_MV_3 19 0 2 diff --git a/tests/queries/0_stateless/02174_cte_scalar_cache_mv.sql b/tests/queries/0_stateless/02174_cte_scalar_cache_mv.sql new file mode 100644 index 00000000000..4d4447c7f31 --- /dev/null +++ b/tests/queries/0_stateless/02174_cte_scalar_cache_mv.sql @@ -0,0 +1,133 @@ +-- TEST CACHE +CREATE TABLE t1 (i Int64, j Int64) ENGINE = Memory; +INSERT INTO t1 SELECT number, number FROM system.numbers LIMIT 100; +CREATE TABLE t2 (k Int64, l Int64, m Int64, n Int64) ENGINE = Memory; + +CREATE MATERIALIZED VIEW mv1 TO t2 AS + WITH + (SELECT max(i) FROM t1) AS t1 + SELECT + t1 as k, -- Using local cache x 4 + t1 as l, + t1 as m, + t1 as n + FROM t1 + LIMIT 5; + +-- FIRST INSERT +INSERT INTO t1 +WITH + (SELECT max(i) FROM t1) AS t1 +SELECT + number as i, + t1 + t1 + t1 AS j -- Using global cache +FROM system.numbers +LIMIT 100 +SETTINGS + min_insert_block_size_rows=5, + max_insert_block_size=5, + min_insert_block_size_rows_for_materialized_views=5, + max_block_size=5, + max_threads=1; + +SELECT k, l, m, n, count() +FROM t2 +GROUP BY k, l, m, n +ORDER BY k, l, m, n; + +SYSTEM FLUSH LOGS; +-- The main query should have a cache miss and 3 global hits +-- The MV is executed 20 times (100 / 5) and each run does 1 miss and 4 hits to the LOCAL cache +-- In addition to this, to prepare the MV, there is an extra preparation to get the list of columns via +-- InterpreterSelectQuery, which adds 1 miss and 4 global hits (since it uses the global cache) +-- So in total we have: +-- Main query: 1 miss, 3 global +-- Preparation: 1 miss, 4 global +-- Blocks (20): 20 miss, 0 global, 80 local hits + +-- TOTAL: 22 miss, 7 global, 80 local +SELECT + '02177_MV', + ProfileEvents['ScalarSubqueriesGlobalCacheHit'] as scalar_cache_global_hit, + ProfileEvents['ScalarSubqueriesLocalCacheHit'] as scalar_cache_local_hit, + ProfileEvents['ScalarSubqueriesCacheMiss'] as scalar_cache_miss +FROM system.query_log +WHERE + current_database = currentDatabase() + AND type = 'QueryFinish' + AND query LIKE '-- FIRST INSERT\nINSERT INTO t1\n%' + AND event_date >= yesterday() AND event_time > now() - interval 10 minute; + +DROP TABLE mv1; + +CREATE TABLE t3 (z Int64) ENGINE = Memory; +CREATE MATERIALIZED VIEW mv2 TO t3 AS +SELECT + -- This includes an unnecessarily complex query to verify that the local cache is used (since it uses t1) + sum(i) + sum(j) + (SELECT * FROM (SELECT min(i) + min(j) FROM (SELECT * FROM system.one _a, t1 _b))) AS z +FROM t1; + +-- SECOND INSERT +INSERT INTO t1 +SELECT 0 as i, number as j from numbers(100) +SETTINGS + min_insert_block_size_rows=5, + max_insert_block_size=5, + min_insert_block_size_rows_for_materialized_views=5, + max_block_size=5, + max_threads=1; + +SELECT * FROM t3 ORDER BY z ASC; +SYSTEM FLUSH LOGS; +SELECT + '02177_MV_2', + ProfileEvents['ScalarSubqueriesGlobalCacheHit'] as scalar_cache_global_hit, + ProfileEvents['ScalarSubqueriesLocalCacheHit'] as scalar_cache_local_hit, + ProfileEvents['ScalarSubqueriesCacheMiss'] as scalar_cache_miss +FROM system.query_log +WHERE + current_database = currentDatabase() + AND type = 'QueryFinish' + AND query LIKE '-- SECOND INSERT\nINSERT INTO t1%' + AND event_date >= yesterday() AND event_time > now() - interval 10 minute; + +DROP TABLE mv2; + + +CREATE TABLE t4 (z Int64) ENGINE = Memory; +CREATE MATERIALIZED VIEW mv3 TO t4 AS +SELECT + -- This includes an unnecessarily complex query but now it uses t2 so it can be cached + min(i) + min(j) + (SELECT * FROM (SELECT min(k) + min(l) FROM (SELECT * FROM system.one _a, t2 _b))) AS z +FROM t1; + +-- THIRD INSERT +INSERT INTO t1 +SELECT number as i, number as j from numbers(100) + SETTINGS + min_insert_block_size_rows=5, + max_insert_block_size=5, + min_insert_block_size_rows_for_materialized_views=5, + max_block_size=5, + max_threads=1; +SYSTEM FLUSH LOGS; + +SELECT * FROM t4 ORDER BY z ASC; + +SELECT + '02177_MV_3', + ProfileEvents['ScalarSubqueriesGlobalCacheHit'] as scalar_cache_global_hit, + ProfileEvents['ScalarSubqueriesLocalCacheHit'] as scalar_cache_local_hit, + ProfileEvents['ScalarSubqueriesCacheMiss'] as scalar_cache_miss +FROM system.query_log +WHERE + current_database = currentDatabase() + AND type = 'QueryFinish' + AND query LIKE '-- THIRD INSERT\nINSERT INTO t1%' + AND event_date >= yesterday() AND event_time > now() - interval 10 minute; + +DROP TABLE mv3; +DROP TABLE t1; +DROP TABLE t2; +DROP TABLE t3; +DROP TABLE t4; diff --git a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference index 645cec31b47..5e8c7fc243f 100644 --- a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference +++ b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.reference @@ -5,4 +5,5 @@ -- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" -- at first and after -- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" -select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings optimize_aggregation_in_order=1; +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key; +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings distributed_aggregation_memory_efficient=0; diff --git a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql index a86fd4357c8..35731c63f0d 100644 --- a/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql +++ b/tests/queries/0_stateless/02176_optimize_aggregation_in_order_empty.sql @@ -1,6 +1,8 @@ drop table if exists data_02176; create table data_02176 (key Int) Engine=MergeTree() order by key; +set optimize_aggregation_in_order=1; + -- { echoOn } -- regression for optimize_aggregation_in_order with empty result set @@ -8,7 +10,8 @@ create table data_02176 (key Int) Engine=MergeTree() order by key; -- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" -- at first and after -- "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" -select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings optimize_aggregation_in_order=1; +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key; +select count() from remote('127.{1,2}', currentDatabase(), data_02176) where key = 0 group by key settings distributed_aggregation_memory_efficient=0; -- { echoOff } drop table data_02176; diff --git a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference index 00e893213c0..0345e05303c 100644 --- a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference +++ b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.reference @@ -2,5 +2,7 @@ -- regression for optimize_aggregation_in_order -- that cause "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" error -select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings optimize_aggregation_in_order=1; +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key; +2 +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings distributed_aggregation_memory_efficient=0; 2 diff --git a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql index 17c4a1dba29..9dc3a24213e 100644 --- a/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql +++ b/tests/queries/0_stateless/02177_merge_optimize_aggregation_in_order.sql @@ -2,11 +2,14 @@ drop table if exists data_02177; create table data_02177 (key Int) Engine=MergeTree() order by key; insert into data_02177 values (1); +set optimize_aggregation_in_order=1; + -- { echoOn } -- regression for optimize_aggregation_in_order -- that cause "Chunk should have AggregatedChunkInfo in GroupingAggregatedTransform" error -select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings optimize_aggregation_in_order=1; +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key; +select count() from remote('127.{1,2}', currentDatabase(), data_02177) group by key settings distributed_aggregation_memory_efficient=0; -- { echoOff } drop table data_02177; diff --git a/tests/queries/0_stateless/02179_dict_reload_on_cluster.reference b/tests/queries/0_stateless/02179_dict_reload_on_cluster.reference new file mode 100644 index 00000000000..a33889cb341 --- /dev/null +++ b/tests/queries/0_stateless/02179_dict_reload_on_cluster.reference @@ -0,0 +1,10 @@ +0 +10 +1 +SYSTEM RELOAD DICTIONARIES ON CLUSTER test_shard_localhost +localhost 9000 0 0 0 +0 +11 +1 +CREATE DATABASE +1 diff --git a/tests/queries/0_stateless/02179_dict_reload_on_cluster.sql b/tests/queries/0_stateless/02179_dict_reload_on_cluster.sql new file mode 100644 index 00000000000..686025acbf8 --- /dev/null +++ b/tests/queries/0_stateless/02179_dict_reload_on_cluster.sql @@ -0,0 +1,40 @@ +-- Tags: no-parallel + +DROP DATABASE IF EXISTS dict_db_02179; +CREATE DATABASE dict_db_02179; + +CREATE TABLE dict_db_02179.dict_data (key UInt64, val UInt64) Engine=Memory(); +CREATE DICTIONARY dict_db_02179.dict +( + key UInt64 DEFAULT 0, + val UInt64 DEFAULT 10 +) +PRIMARY KEY key +SOURCE(CLICKHOUSE(HOST 'localhost' PORT tcpPort() USER 'default' TABLE 'dict_data' PASSWORD '' DB 'dict_db_02179')) +LIFETIME(MIN 0 MAX 0) +LAYOUT(FLAT()); + +INSERT INTO dict_db_02179.dict_data VALUES(1,11); + +SELECT query_count FROM system.dictionaries WHERE database = 'dict_db_02179' AND name = 'dict'; +SELECT dictGetUInt64('dict_db_02179.dict', 'val', toUInt64(0)); +SELECT query_count FROM system.dictionaries WHERE database = 'dict_db_02179' AND name = 'dict'; + + +SELECT 'SYSTEM RELOAD DICTIONARIES ON CLUSTER test_shard_localhost'; +SET distributed_ddl_output_mode='throw'; +SYSTEM RELOAD DICTIONARIES ON CLUSTER test_shard_localhost; +SET distributed_ddl_output_mode='none'; +SELECT query_count FROM system.dictionaries WHERE database = 'dict_db_02179' AND name = 'dict'; +SELECT dictGetUInt64('dict_db_02179.dict', 'val', toUInt64(1)); +SELECT query_count FROM system.dictionaries WHERE database = 'dict_db_02179' AND name = 'dict'; + +SELECT 'CREATE DATABASE'; +DROP DATABASE IF EXISTS empty_db_02179; +CREATE DATABASE empty_db_02179; +SELECT query_count FROM system.dictionaries WHERE database = 'dict_db_02179' AND name = 'dict'; + +DROP DICTIONARY dict_db_02179.dict; +DROP TABLE dict_db_02179.dict_data; +DROP DATABASE dict_db_02179; +DROP DATABASE empty_db_02179; diff --git a/tests/queries/0_stateless/02180_group_by_lowcardinality.reference b/tests/queries/0_stateless/02180_group_by_lowcardinality.reference new file mode 100644 index 00000000000..a7149390d1a --- /dev/null +++ b/tests/queries/0_stateless/02180_group_by_lowcardinality.reference @@ -0,0 +1,10 @@ +{"val":"1563.8","avg(toUInt32(val))":null} +{"val":"891.4","avg(toUInt32(val))":null} +{"val":"584.4","avg(toUInt32(val))":null} +{"val":"269","avg(toUInt32(val))":269} +{"val":"1233.4","avg(toUInt32(val))":null} +{"val":"1833","avg(toUInt32(val))":1833} +{"val":"1009.4","avg(toUInt32(val))":null} +{"val":"1178.6","avg(toUInt32(val))":null} +{"val":"372.6","avg(toUInt32(val))":null} +{"val":"232.4","avg(toUInt32(val))":null} diff --git a/tests/queries/0_stateless/02180_group_by_lowcardinality.sql b/tests/queries/0_stateless/02180_group_by_lowcardinality.sql new file mode 100644 index 00000000000..463753a624e --- /dev/null +++ b/tests/queries/0_stateless/02180_group_by_lowcardinality.sql @@ -0,0 +1,10 @@ +create table if not exists t_group_by_lowcardinality(p_date Date, val LowCardinality(Nullable(String))) +engine=MergeTree() partition by p_date order by tuple(); + +insert into t_group_by_lowcardinality select today() as p_date, toString(number/5) as val from numbers(10000); +insert into t_group_by_lowcardinality select today() as p_date, Null as val from numbers(100); + +select val, avg(toUInt32(val)) from t_group_by_lowcardinality group by val limit 10 settings max_threads=1, max_rows_to_group_by=100, group_by_overflow_mode='any' format JSONEachRow; + +drop table if exists t_group_by_lowcardinality; + diff --git a/tests/queries/0_stateless/02182_format_and_schema_from_stdin.reference b/tests/queries/0_stateless/02182_format_and_schema_from_stdin.reference new file mode 100644 index 00000000000..145e66f828c --- /dev/null +++ b/tests/queries/0_stateless/02182_format_and_schema_from_stdin.reference @@ -0,0 +1,11 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +1 String [1,2,3] diff --git a/tests/queries/0_stateless/02182_format_and_schema_from_stdin.sh b/tests/queries/0_stateless/02182_format_and_schema_from_stdin.sh new file mode 100755 index 00000000000..6d0de1f1cf8 --- /dev/null +++ b/tests/queries/0_stateless/02182_format_and_schema_from_stdin.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-parallel, no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT -q "select * from numbers(10) format Parquet" > $CLICKHOUSE_TMP/data.parquet +$CLICKHOUSE_LOCAL -q "select * from table" --file="-" < $CLICKHOUSE_TMP/data.parquet + +echo "1,\"String\", \"[1, 2, 3]\"" | $CLICKHOUSE_LOCAL -q "select * from table" --input-format=CSV + diff --git a/tests/queries/0_stateless/02183_array_tuple_literals_remote.reference b/tests/queries/0_stateless/02183_array_tuple_literals_remote.reference new file mode 100644 index 00000000000..1444ec4d2d9 --- /dev/null +++ b/tests/queries/0_stateless/02183_array_tuple_literals_remote.reference @@ -0,0 +1,11 @@ +[0] +(0,1) +[[0,1],[2,3]] +[(0,1),(2,3)] +[(0,1),(2,3)] +([0,1],(2,3),[4],(5,'a'),6,'b') +[0,1] +(0,1) +[[0,1],[2,3]] +[[0,1],[0,0]] +[[[0]],[[1],[2,3]]] diff --git a/tests/queries/0_stateless/02183_array_tuple_literals_remote.sql b/tests/queries/0_stateless/02183_array_tuple_literals_remote.sql new file mode 100644 index 00000000000..25c7e7d7348 --- /dev/null +++ b/tests/queries/0_stateless/02183_array_tuple_literals_remote.sql @@ -0,0 +1,11 @@ +SELECT any(array(0)) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(tuple(0, 1)) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(array(array(0, 1), array(2, 3))) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(array(tuple(0, 1), tuple(2, 3))) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(array((0, 1), (2, 3))) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(tuple(array(0, 1), tuple(2, 3), [4], (5, 'a'), 6, 'b')) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(array(number, 1)) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(tuple(number, 1)) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(array(array(0, 1), [2, 3])) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any(array(array(0, 1), [number, number])) AS k FROM remote('127.0.0.{1,2}', numbers(10)); +SELECT any([[[number]],[[number + 1], [number + 2, number + 3]]]) AS k FROM remote('127.0.0.{1,2}', numbers(10)); diff --git a/tests/queries/0_stateless/02184_default_table_engine.reference b/tests/queries/0_stateless/02184_default_table_engine.reference new file mode 100644 index 00000000000..200578f3da9 --- /dev/null +++ b/tests/queries/0_stateless/02184_default_table_engine.reference @@ -0,0 +1,27 @@ +CREATE TABLE default.table_02184\n(\n `x` UInt8\n)\nENGINE = Log +CREATE TABLE default.table_02184\n(\n `x` UInt8\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +CREATE TABLE default.test_optimize_exception\n(\n `date` Date\n)\nENGINE = MergeTree\nPARTITION BY toYYYYMM(date)\nORDER BY date\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_02184\n(\n `x` UInt8\n)\nENGINE = MergeTree\nORDER BY x\nSETTINGS index_granularity = 8192 +CREATE TABLE default.table_02184\n(\n `x` UInt8\n)\nENGINE = MergeTree\nPRIMARY KEY x\nORDER BY x\nSETTINGS index_granularity = 8192 +CREATE TABLE default.numbers1\n(\n `number` UInt64\n)\nENGINE = Memory +4.5 +45 +CREATE TABLE default.numbers2\n(\n `number` UInt64\n)\nENGINE = MergeTree\nORDER BY intHash32(number)\nSAMPLE BY intHash32(number)\nSETTINGS index_granularity = 8192 +45 +CREATE TABLE default.numbers3\n(\n `number` UInt64\n)\nENGINE = Log +CREATE MATERIALIZED VIEW default.test_view_filtered\n(\n `EventDate` Date,\n `CounterID` UInt32\n)\nENGINE = Memory AS\nSELECT\n CounterID,\n EventDate\nFROM default.test_table\nWHERE EventDate < \'2013-01-01\' +2014-01-02 0 0 1970-01-01 03:00:00 2014-01-02 03:04:06 +1 2014-01-02 07:04:06 +CREATE TABLE default.t1\n(\n `Rows` UInt64,\n `MaxHitTime` DateTime(\'UTC\')\n)\nENGINE = MergeTree\nORDER BY Rows\nSETTINGS index_granularity = 8192 +CREATE TABLE default.t2\n(\n `Rows` UInt64,\n `MaxHitTime` DateTime(\'UTC\')\n)\nENGINE = Memory +CREATE TABLE default.mt\n(\n `a` UInt64,\n `b` Nullable(String),\n INDEX b_index b TYPE set(123) GRANULARITY 1\n)\nENGINE = MergeTree\nPRIMARY KEY (a, coalesce(b, \'test\'))\nORDER BY (a, coalesce(b, \'test\'))\nSETTINGS index_granularity = 8192 +CREATE TABLE default.mt2\n(\n `a` UInt64,\n `b` Nullable(String),\n INDEX b_index b TYPE set(123) GRANULARITY 1\n)\nENGINE = MergeTree\nPRIMARY KEY (a, coalesce(b, \'test\'))\nORDER BY (a, coalesce(b, \'test\'))\nSETTINGS index_granularity = 8192 +CREATE TEMPORARY TABLE tmp\n(\n `n` Int32\n)\nENGINE = Memory +CREATE TABLE default.log\n(\n `n` Int32\n)\nENGINE = Log +CREATE TABLE default.log1\n(\n `n` Int32\n)\nENGINE = Log +CREATE TABLE default.mem\n(\n `n` Int32\n)\nENGINE = Memory +CREATE TABLE default.mt\n(\n `n` UInt8\n)\nENGINE = MergeTree\nORDER BY n\nSETTINGS index_granularity = 8192 +CREATE TABLE default.mem\n(\n `n` UInt8\n)\nENGINE = Memory +CREATE TABLE default.val\n(\n `n` Int32\n) AS values(\'n int\', 1, 2) +CREATE TABLE default.val2\n(\n `n` Int32\n) AS values(\'n int\', 1, 2) +CREATE TABLE default.log\n(\n `n` Int32\n)\nENGINE = Log diff --git a/tests/queries/0_stateless/02184_default_table_engine.sql b/tests/queries/0_stateless/02184_default_table_engine.sql new file mode 100644 index 00000000000..d129ccc801e --- /dev/null +++ b/tests/queries/0_stateless/02184_default_table_engine.sql @@ -0,0 +1,120 @@ +CREATE TABLE table_02184 (x UInt8); --{serverError 119} +SET default_table_engine = 'Log'; +CREATE TABLE table_02184 (x UInt8); +SHOW CREATE TABLE table_02184; +DROP TABLE table_02184; + +SET default_table_engine = 'MergeTree'; +CREATE TABLE table_02184 (x UInt8); --{serverError 42} +CREATE TABLE table_02184 (x UInt8, PRIMARY KEY (x)); +SHOW CREATE TABLE table_02184; +DROP TABLE table_02184; + +CREATE TABLE test_optimize_exception (date Date) PARTITION BY toYYYYMM(date) ORDER BY date; +SHOW CREATE TABLE test_optimize_exception; +DROP TABLE test_optimize_exception; +CREATE TABLE table_02184 (x UInt8) PARTITION BY x; --{serverError 36} +CREATE TABLE table_02184 (x UInt8) ORDER BY x; +SHOW CREATE TABLE table_02184; +DROP TABLE table_02184; + +CREATE TABLE table_02184 (x UInt8) PRIMARY KEY x; +SHOW CREATE TABLE table_02184; +DROP TABLE table_02184; +SET default_table_engine = 'Memory'; +CREATE TABLE numbers1 AS SELECT number FROM numbers(10); +SHOW CREATE TABLE numbers1; +SELECT avg(number) FROM numbers1; +DROP TABLE numbers1; + +SET default_table_engine = 'MergeTree'; +CREATE TABLE numbers2 ORDER BY intHash32(number) SAMPLE BY intHash32(number) AS SELECT number FROM numbers(10); +SELECT sum(number) FROM numbers2; +SHOW CREATE TABLE numbers2; +DROP TABLE numbers2; + +CREATE TABLE numbers3 ENGINE = Log AS SELECT number FROM numbers(10); +SELECT sum(number) FROM numbers3; +SHOW CREATE TABLE numbers3; +DROP TABLE numbers3; + +CREATE TABLE test_table (EventDate Date, CounterID UInt32, UserID UInt64, EventTime DateTime('Europe/Moscow'), UTCEventTime DateTime('UTC')) PARTITION BY EventDate PRIMARY KEY CounterID; +SET default_table_engine = 'Memory'; +CREATE MATERIALIZED VIEW test_view (Rows UInt64, MaxHitTime DateTime('Europe/Moscow')) AS SELECT count() AS Rows, max(UTCEventTime) AS MaxHitTime FROM test_table; +CREATE MATERIALIZED VIEW test_view_filtered (EventDate Date, CounterID UInt32) POPULATE AS SELECT CounterID, EventDate FROM test_table WHERE EventDate < '2013-01-01'; +SHOW CREATE TABLE test_view_filtered; +INSERT INTO test_table (EventDate, UTCEventTime) VALUES ('2014-01-02', '2014-01-02 03:04:06'); + +SELECT * FROM test_table; +SELECT * FROM test_view; +SELECT * FROM test_view_filtered; + +DROP TABLE test_view; +DROP TABLE test_view_filtered; + +SET default_table_engine = 'MergeTree'; +CREATE MATERIALIZED VIEW test_view ORDER BY Rows AS SELECT count() AS Rows, max(UTCEventTime) AS MaxHitTime FROM test_table; +SET default_table_engine = 'Memory'; +CREATE TABLE t1 AS test_view; +CREATE TABLE t2 ENGINE=Memory AS test_view; +SHOW CREATE TABLE t1; +SHOW CREATE TABLE t2; +DROP TABLE test_view; +DROP TABLE test_table; +DROP TABLE t1; +DROP TABLE t2; + + +CREATE DATABASE test_02184 ORDER BY kek; -- {serverError 80} +CREATE DATABASE test_02184 SETTINGS x=1; -- {serverError 80} +CREATE TABLE table_02184 (x UInt8, y int, PRIMARY KEY (x)) ENGINE=MergeTree PRIMARY KEY y; -- {clientError 36} +SET default_table_engine = 'MergeTree'; +CREATE TABLE table_02184 (x UInt8, y int, PRIMARY KEY (x)) PRIMARY KEY y; -- {clientError 36} + +CREATE TABLE mt (a UInt64, b Nullable(String), PRIMARY KEY (a, coalesce(b, 'test')), INDEX b_index b TYPE set(123) GRANULARITY 1); +SHOW CREATE TABLE mt; +SET default_table_engine = 'Log'; +CREATE TABLE mt2 AS mt; +SHOW CREATE TABLE mt2; +DROP TABLE mt; + +SET default_table_engine = 'Log'; +CREATE TEMPORARY TABLE tmp (n int); +SHOW CREATE TEMPORARY TABLE tmp; +CREATE TEMPORARY TABLE tmp1 (n int) ENGINE=Memory; +CREATE TEMPORARY TABLE tmp2 (n int) ENGINE=Log; -- {serverError 80} +CREATE TEMPORARY TABLE tmp2 (n int) ORDER BY n; -- {serverError 80} +CREATE TEMPORARY TABLE tmp2 (n int, PRIMARY KEY (n)); -- {serverError 80} + +CREATE TABLE log (n int); +SHOW CREATE log; +SET default_table_engine = 'MergeTree'; +CREATE TABLE log1 AS log; +SHOW CREATE log1; +CREATE TABLE mem AS log1 ENGINE=Memory; +SHOW CREATE mem; +DROP TABLE log; +DROP TABLE log1; +DROP TABLE mem; + +SET default_table_engine = 'None'; +CREATE TABLE mem AS SELECT 1 as n; --{serverError 119} +SET default_table_engine = 'Memory'; +CREATE TABLE mem ORDER BY n AS SELECT 1 as n; -- {serverError 36} +SET default_table_engine = 'MergeTree'; +CREATE TABLE mt ORDER BY n AS SELECT 1 as n; +CREATE TABLE mem ENGINE=Memory AS SELECT 1 as n; +SHOW CREATE TABLE mt; +SHOW CREATE TABLE mem; +DROP TABLE mt; +DROP TABLE mem; + +CREATE TABLE val AS values('n int', 1, 2); +CREATE TABLE val2 AS val; +CREATE TABLE log ENGINE=Log AS val; +SHOW CREATE TABLE val; +SHOW CREATE TABLE val2; +SHOW CREATE TABLE log; +DROP TABLE val; +DROP TABLE val2; +DROP TABLE log; diff --git a/tests/queries/0_stateless/02184_nested_tuple.reference b/tests/queries/0_stateless/02184_nested_tuple.reference new file mode 100644 index 00000000000..b435e2f28a6 --- /dev/null +++ b/tests/queries/0_stateless/02184_nested_tuple.reference @@ -0,0 +1,7 @@ +{"endUserIDs":{"_experience":{"aaid":{"id":"id_1","namespace":{"code":"code_1"},"primary":1},"mcid":{"id":"id_2","namespace":{"code":"code_2"},"primary":2}}}} +{"endUserIDs._experience":{"aaid":{"id":"id_1","namespace":{"code":"code_1"},"primary":1},"mcid":{"id":"id_2","namespace":{"code":"code_2"},"primary":2}}} +{"endUserIDs._experience.aaid":{"id":"id_1","namespace":{"code":"code_1"},"primary":1}} +{"endUserIDs._experience.aaid.id":"id_1"} +{"endUserIDs._experience.aaid.namespace":{"code":"code_1"}} +{"endUserIDs._experience.aaid.namespace.code":"code_1"} +{"endUserIDs._experience.aaid.primary":1} diff --git a/tests/queries/0_stateless/02184_nested_tuple.sql b/tests/queries/0_stateless/02184_nested_tuple.sql new file mode 100644 index 00000000000..67a20e3dce1 --- /dev/null +++ b/tests/queries/0_stateless/02184_nested_tuple.sql @@ -0,0 +1,38 @@ +DROP TABLE IF EXISTS t_nested_tuple; + +CREATE TABLE t_nested_tuple +( + endUserIDs Tuple( + _experience Tuple( + aaid Tuple( + id Nullable(String), + namespace Tuple( + code LowCardinality(Nullable(String)) + ), + primary LowCardinality(Nullable(UInt8)) + ), + mcid Tuple( + id Nullable(String), + namespace Tuple( + code LowCardinality(Nullable(String)) + ), + primary LowCardinality(Nullable(UInt8)) + ) + ) + ) +) +ENGINE = MergeTree ORDER BY tuple(); + +SET output_format_json_named_tuples_as_objects = 1; + +INSERT INTO t_nested_tuple FORMAT JSONEachRow {"endUserIDs":{"_experience":{"aaid":{"id":"id_1","namespace":{"code":"code_1"},"primary":1},"mcid":{"id":"id_2","namespace":{"code":"code_2"},"primary":2}}}}; + +SELECT * FROM t_nested_tuple FORMAT JSONEachRow; +SELECT endUserIDs._experience FROM t_nested_tuple FORMAT JSONEachRow; +SELECT endUserIDs._experience.aaid FROM t_nested_tuple FORMAT JSONEachRow; +SELECT endUserIDs._experience.aaid.id FROM t_nested_tuple FORMAT JSONEachRow; +SELECT endUserIDs._experience.aaid.namespace FROM t_nested_tuple FORMAT JSONEachRow; +SELECT endUserIDs._experience.aaid.namespace.code FROM t_nested_tuple FORMAT JSONEachRow; +SELECT endUserIDs._experience.aaid.primary FROM t_nested_tuple FORMAT JSONEachRow; + +DROP TABLE t_nested_tuple; diff --git a/tests/queries/0_stateless/02184_table_engine_access.reference b/tests/queries/0_stateless/02184_table_engine_access.reference new file mode 100644 index 00000000000..99a67d4daa4 --- /dev/null +++ b/tests/queries/0_stateless/02184_table_engine_access.reference @@ -0,0 +1,2 @@ +ACCESS_DENIED +CREATE TABLE default.t\n(\n `line` String\n)\nENGINE = URL(\'https://clickhouse.com\', \'LineAsString\') diff --git a/tests/queries/0_stateless/02184_table_engine_access.sh b/tests/queries/0_stateless/02184_table_engine_access.sh new file mode 100755 index 00000000000..dbbf28e46d4 --- /dev/null +++ b/tests/queries/0_stateless/02184_table_engine_access.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Tags: no-parallel +# Tag no-parallel: create user + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT --query "DROP USER IF EXISTS user_test_02184;" +$CLICKHOUSE_CLIENT --query "CREATE USER user_test_02184 IDENTIFIED WITH plaintext_password BY 'user_test_02184';" +${CLICKHOUSE_CLIENT} -q "REVOKE ALL ON *.* FROM user_test_02184" + +$CLICKHOUSE_CLIENT --query "GRANT CREATE ON *.* TO user_test_02184;" + +$CLICKHOUSE_CLIENT --query "CREATE TABLE url ENGINE=URL('https://clickhouse.com', LineAsString)" + +$CLICKHOUSE_CLIENT --user=user_test_02184 --password=user_test_02184 --query "CREATE TABLE t AS url" 2>&1| grep -Fo "ACCESS_DENIED" | uniq + +$CLICKHOUSE_CLIENT --query "GRANT URL ON *.* TO user_test_02184;" +$CLICKHOUSE_CLIENT --user=user_test_02184 --password=user_test_02184 --query "CREATE TABLE t AS url" +$CLICKHOUSE_CLIENT --query "SHOW CREATE TABLE t" +$CLICKHOUSE_CLIENT --query "DROP TABLE t" +$CLICKHOUSE_CLIENT --query "DROP TABLE url" diff --git a/tests/queries/0_stateless/02185_orc_corrupted_file.reference b/tests/queries/0_stateless/02185_orc_corrupted_file.reference new file mode 100644 index 00000000000..d86bac9de59 --- /dev/null +++ b/tests/queries/0_stateless/02185_orc_corrupted_file.reference @@ -0,0 +1 @@ +OK diff --git a/tests/queries/0_stateless/02185_orc_corrupted_file.sh b/tests/queries/0_stateless/02185_orc_corrupted_file.sh new file mode 100755 index 00000000000..7d7a714cccc --- /dev/null +++ b/tests/queries/0_stateless/02185_orc_corrupted_file.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER_FILES_PATH=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}') +cp $CUR_DIR/data_orc/corrupted.orc $USER_FILES_PATH/ + +${CLICKHOUSE_CLIENT} --query="select * from file('corrupted.orc')" 2>&1 | grep -F -q 'CANNOT_EXTRACT_TABLE_STRUCTURE' && echo 'OK' || echo 'FAIL' + diff --git a/tests/queries/0_stateless/02185_split_by_char.reference b/tests/queries/0_stateless/02185_split_by_char.reference new file mode 100644 index 00000000000..f69d8d35006 --- /dev/null +++ b/tests/queries/0_stateless/02185_split_by_char.reference @@ -0,0 +1,5 @@ +['1','2','3'] +['1,2,3'] +['1','2,3'] +['1','2','3'] +['1','2','3'] diff --git a/tests/queries/0_stateless/02185_split_by_char.sql b/tests/queries/0_stateless/02185_split_by_char.sql new file mode 100644 index 00000000000..6c490654813 --- /dev/null +++ b/tests/queries/0_stateless/02185_split_by_char.sql @@ -0,0 +1,8 @@ +select splitByChar(',', '1,2,3'); +select splitByChar(',', '1,2,3', 0); +select splitByChar(',', '1,2,3', 1); +select splitByChar(',', '1,2,3', 2); +select splitByChar(',', '1,2,3', 3); + +select splitByChar(',', '1,2,3', -2); -- { serverError 44 } +select splitByChar(',', '1,2,3', ''); -- { serverError 43 } \ No newline at end of file diff --git a/tests/queries/0_stateless/02185_values_schema_inference.reference b/tests/queries/0_stateless/02185_values_schema_inference.reference new file mode 100644 index 00000000000..5353d083b9e --- /dev/null +++ b/tests/queries/0_stateless/02185_values_schema_inference.reference @@ -0,0 +1,14 @@ +String +abc +def +1 +-1 +10000 +-10000 +1000000 +1 string [1,2,-1] +-10 def [10,20,10000] +1 \N [1,2,-1] +\N def [10,NULL,10000] +(1,'1') 10 +(-1,'-1') 1000000 diff --git a/tests/queries/0_stateless/02185_values_schema_inference.sh b/tests/queries/0_stateless/02185_values_schema_inference.sh new file mode 100755 index 00000000000..3cfcd0bd4e6 --- /dev/null +++ b/tests/queries/0_stateless/02185_values_schema_inference.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Tags: no-parallel, no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_CLIENT -q "select * from values('String', 'abc', 'def')" +$CLICKHOUSE_CLIENT -q "select * from values(1, -1, 10000, -10000, 1000000)" +$CLICKHOUSE_CLIENT -q "select * from values((1, 'string', [1, 2, -1]), (-10, 'def', [10, 20, 10000]))" +$CLICKHOUSE_CLIENT -q "select * from values((1, NULL, [1, 2, -1]), (NULL, 'def', [10, NULL, 10000]))" +$CLICKHOUSE_CLIENT -q "select * from values(((1, '1'), 10), ((-1, '-1'), 1000000))" + diff --git a/tests/queries/0_stateless/02187_async_inserts_all_formats.python b/tests/queries/0_stateless/02187_async_inserts_all_formats.python new file mode 100644 index 00000000000..0a909451259 --- /dev/null +++ b/tests/queries/0_stateless/02187_async_inserts_all_formats.python @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +import os +import sys + +CURDIR = os.path.dirname(os.path.realpath(__file__)) +sys.path.insert(0, os.path.join(CURDIR, 'helpers')) + +CLICKHOUSE_URL = os.environ.get('CLICKHOUSE_URL') +CLICKHOUSE_TMP = os.environ.get('CLICKHOUSE_TMP') + +from pure_http_client import ClickHouseClient + +client = ClickHouseClient() + +def run_test(data_format, gen_data_template, settings): + print(data_format) + client.query("TRUNCATE TABLE t_async_insert") + + expected = client.query(gen_data_template.format("TSV")).strip() + data = client.query(gen_data_template.format(data_format), settings=settings,binary_result=True) + + insert_query = "INSERT INTO t_async_insert FORMAT {}".format(data_format) + client.query_with_data(insert_query, data, settings=settings) + + result = client.query("SELECT * FROM t_async_insert FORMAT TSV").strip() + if result != expected: + print("Failed for format {}.\nExpected:\n{}\nGot:\n{}\n".format(data_format, expected, result)) + exit(1) + +formats = client.query("SELECT name FROM system.formats WHERE is_input AND is_output \ + AND name NOT IN ('CapnProto', 'RawBLOB', 'Template', 'ProtobufSingle', 'LineAsString', 'Protobuf') ORDER BY name").strip().split('\n') + +# Generic formats +client.query("DROP TABLE IF EXISTS t_async_insert") +client.query("CREATE TABLE t_async_insert (id UInt64, s String, arr Array(UInt64)) ENGINE = Memory") +gen_data_query = "SELECT number AS id, toString(number) AS s, range(number) AS arr FROM numbers(10) FORMAT {}" + +for data_format in formats: + run_test(data_format, gen_data_query, settings={"async_insert": 1, "wait_for_async_insert": 1}) + +# LineAsString +client.query("DROP TABLE IF EXISTS t_async_insert") +client.query("CREATE TABLE t_async_insert (s String) ENGINE = Memory") +gen_data_query = "SELECT toString(number) AS s FROM numbers(10) FORMAT {}" + +run_test('LineAsString', gen_data_query, settings={"async_insert": 1, "wait_for_async_insert": 1}) + +# TODO: add CapnProto and Protobuf + +print("OK") diff --git a/tests/queries/0_stateless/02187_async_inserts_all_formats.reference b/tests/queries/0_stateless/02187_async_inserts_all_formats.reference new file mode 100644 index 00000000000..b4a5b6c3a42 --- /dev/null +++ b/tests/queries/0_stateless/02187_async_inserts_all_formats.reference @@ -0,0 +1,40 @@ +Arrow +ArrowStream +Avro +CSV +CSVWithNames +CSVWithNamesAndTypes +CustomSeparated +CustomSeparatedWithNames +CustomSeparatedWithNamesAndTypes +JSONCompactEachRow +JSONCompactEachRowWithNames +JSONCompactEachRowWithNamesAndTypes +JSONCompactStringsEachRow +JSONCompactStringsEachRowWithNames +JSONCompactStringsEachRowWithNamesAndTypes +JSONEachRow +JSONStringsEachRow +MsgPack +Native +ORC +Parquet +RowBinary +RowBinaryWithNames +RowBinaryWithNamesAndTypes +TSKV +TSV +TSVRaw +TSVRawWithNames +TSVRawWithNamesAndTypes +TSVWithNames +TSVWithNamesAndTypes +TabSeparated +TabSeparatedRaw +TabSeparatedRawWithNames +TabSeparatedRawWithNamesAndTypes +TabSeparatedWithNames +TabSeparatedWithNamesAndTypes +Values +LineAsString +OK diff --git a/tests/queries/0_stateless/02187_async_inserts_all_formats.sh b/tests/queries/0_stateless/02187_async_inserts_all_formats.sh new file mode 100755 index 00000000000..4b0b8d84c58 --- /dev/null +++ b/tests/queries/0_stateless/02187_async_inserts_all_formats.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# We should have correct env vars from shell_config.sh to run this test +python3 "$CURDIR"/02187_async_inserts_all_formats.python diff --git a/tests/queries/0_stateless/02187_insert_values_with_mv.reference b/tests/queries/0_stateless/02187_insert_values_with_mv.reference new file mode 100644 index 00000000000..d24ee4faff7 --- /dev/null +++ b/tests/queries/0_stateless/02187_insert_values_with_mv.reference @@ -0,0 +1,12 @@ +VALUES 1 +TABLE 1 +VALUES 1 +VALUES 1 +VALUES 1 +VALUES 1 +VALUES 1 +TABLE 1 +TABLE 1 +TABLE 1 +TABLE 1 +TABLE 1 diff --git a/tests/queries/0_stateless/02187_insert_values_with_mv.sql b/tests/queries/0_stateless/02187_insert_values_with_mv.sql new file mode 100644 index 00000000000..91241f7f5c3 --- /dev/null +++ b/tests/queries/0_stateless/02187_insert_values_with_mv.sql @@ -0,0 +1,59 @@ +CREATE TABLE IF NOT EXISTS a (a Int64) ENGINE=Memory; +CREATE TABLE IF NOT EXISTS b (a Int64) ENGINE=Memory; +CREATE MATERIALIZED VIEW IF NOT EXISTS mv1 TO b AS Select sleepEachRow(0.05) as a FROM a; +CREATE MATERIALIZED VIEW IF NOT EXISTS mv2 TO b AS Select sleepEachRow(0.05) as a FROM a; +CREATE MATERIALIZED VIEW IF NOT EXISTS mv3 TO b AS Select sleepEachRow(0.05) as a FROM a; +CREATE MATERIALIZED VIEW IF NOT EXISTS mv4 TO b AS Select sleepEachRow(0.05) as a FROM a; +CREATE MATERIALIZED VIEW IF NOT EXISTS mv5 TO b AS Select sleepEachRow(0.05) as a FROM a; + +-- INSERT USING VALUES +INSERT INTO a VALUES (1); +-- INSERT USING TABLE +INSERT INTO a SELECT * FROM system.one; +SYSTEM FLUSH LOGS; + +SELECT 'VALUES', query_duration_ms >= 250 +FROM system.query_log +WHERE + current_database = currentDatabase() + AND event_date >= yesterday() + AND query LIKE '-- INSERT USING VALUES%' + AND type = 'QueryFinish' +LIMIT 1; + +SELECT 'TABLE', query_duration_ms >= 250 +FROM system.query_log +WHERE + current_database = currentDatabase() + AND event_date >= yesterday() + AND query LIKE '-- INSERT USING VALUES%' + AND type = 'QueryFinish' +LIMIT 1; + +WITH + ( + SELECT initial_query_id + FROM system.query_log + WHERE + current_database = currentDatabase() + AND event_date >= yesterday() + AND query LIKE '-- INSERT USING VALUES%' + LIMIT 1 + ) AS q_id +SELECT 'VALUES', view_duration_ms >= 50 +FROM system.query_views_log +WHERE initial_query_id = q_id; + +WITH +( + SELECT initial_query_id + FROM system.query_log + WHERE + current_database = currentDatabase() + AND event_date >= yesterday() + AND query LIKE '-- INSERT USING TABLE%' + LIMIT 1 +) AS q_id +SELECT 'TABLE', view_duration_ms >= 50 +FROM system.query_views_log +WHERE initial_query_id = q_id; diff --git a/tests/queries/0_stateless/02187_msg_pack_uuid.reference b/tests/queries/0_stateless/02187_msg_pack_uuid.reference new file mode 100644 index 00000000000..c567cc14ad2 --- /dev/null +++ b/tests/queries/0_stateless/02187_msg_pack_uuid.reference @@ -0,0 +1,4 @@ +5e7084e0-019f-461f-9e70-84e0019f561f +5e7084e0-019f-461f-9e70-84e0019f561f +5e7084e0-019f-461f-9e70-84e0019f561f +5e7084e0-019f-461f-9e70-84e0019f561f UUID diff --git a/tests/queries/0_stateless/02187_msg_pack_uuid.sh b/tests/queries/0_stateless/02187_msg_pack_uuid.sh new file mode 100755 index 00000000000..9be92d66790 --- /dev/null +++ b/tests/queries/0_stateless/02187_msg_pack_uuid.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Tags: no-parallel, no-fasttest + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q "insert into table function file('uuid_str.msgpack', 'MsgPack', 'uuid UUID') select toUUID('5e7084e0-019f-461f-9e70-84e0019f561f') settings output_format_msgpack_uuid_representation='str'" +$CLICKHOUSE_CLIENT -q "select * from file('uuid_str.msgpack', 'MsgPack', 'uuid UUID')" + +$CLICKHOUSE_CLIENT -q "insert into table function file('uuid_bin.msgpack', 'MsgPack', 'uuid UUID') select toUUID('5e7084e0-019f-461f-9e70-84e0019f561f') settings output_format_msgpack_uuid_representation='bin'" +$CLICKHOUSE_CLIENT -q "select * from file('uuid_bin.msgpack', 'MsgPack', 'uuid UUID')" + +$CLICKHOUSE_CLIENT -q "insert into table function file('uuid_ext.msgpack', 'MsgPack', 'uuid UUID') select toUUID('5e7084e0-019f-461f-9e70-84e0019f561f') settings output_format_msgpack_uuid_representation='ext'" +$CLICKHOUSE_CLIENT -q "select * from file('uuid_ext.msgpack', 'MsgPack', 'uuid UUID')" +$CLICKHOUSE_CLIENT -q "select c1, toTypeName(c1) from file('uuid_ext.msgpack') settings input_format_msgpack_number_of_columns=1" + diff --git a/tests/queries/0_stateless/02187_test_final_and_limit_modifier.reference b/tests/queries/0_stateless/02187_test_final_and_limit_modifier.reference new file mode 100644 index 00000000000..56bbfcf090c --- /dev/null +++ b/tests/queries/0_stateless/02187_test_final_and_limit_modifier.reference @@ -0,0 +1,2 @@ +something 1 +something 1 diff --git a/tests/queries/0_stateless/02187_test_final_and_limit_modifier.sql b/tests/queries/0_stateless/02187_test_final_and_limit_modifier.sql new file mode 100644 index 00000000000..7c4ae936865 --- /dev/null +++ b/tests/queries/0_stateless/02187_test_final_and_limit_modifier.sql @@ -0,0 +1,15 @@ +DROP TABLE IF EXISTS test_02187; +CREATE TABLE test_02187 ( + info String, + id Int32 +) +ENGINE = ReplacingMergeTree(id) +ORDER BY id; + +INSERT INTO TABLE test_02187 VALUES ('nothing', 1); +INSERT INTO TABLE test_02187 VALUES ('something', 1); + +SELECT * FROM test_02187 FINAL; +SELECT * FROM test_02187 FINAL LIMIT 1; + + diff --git a/tests/queries/0_stateless/02188_parser_dictionary_primary_key.reference b/tests/queries/0_stateless/02188_parser_dictionary_primary_key.reference new file mode 100644 index 00000000000..0e4e614d264 --- /dev/null +++ b/tests/queries/0_stateless/02188_parser_dictionary_primary_key.reference @@ -0,0 +1,8 @@ +Dictionary output +0 Value +Dictionary output +0 Value +Dictionary output +0 Value +Dictionary output +0 Value diff --git a/tests/queries/0_stateless/02188_parser_dictionary_primary_key.sql b/tests/queries/0_stateless/02188_parser_dictionary_primary_key.sql new file mode 100644 index 00000000000..a939c30b57b --- /dev/null +++ b/tests/queries/0_stateless/02188_parser_dictionary_primary_key.sql @@ -0,0 +1,65 @@ +DROP TABLE IF EXISTS 02188_test_dictionary_source; +CREATE TABLE 02188_test_dictionary_source +( + id UInt64, + value String +) +ENGINE=TinyLog; + +INSERT INTO 02188_test_dictionary_source VALUES (0, 'Value'); + +DROP DICTIONARY IF EXISTS 02188_test_dictionary_simple_primary_key; +CREATE DICTIONARY 02188_test_dictionary_simple_primary_key +( + id UInt64, + value String +) +PRIMARY KEY id +SOURCE(CLICKHOUSE(TABLE '02188_test_dictionary_source')) +LAYOUT(DIRECT()); + +SELECT 'Dictionary output'; +SELECT * FROM 02188_test_dictionary_simple_primary_key; +DROP DICTIONARY 02188_test_dictionary_simple_primary_key; + +CREATE DICTIONARY 02188_test_dictionary_simple_primary_key +( + id UInt64, + value String +) +PRIMARY KEY (id) +SOURCE(CLICKHOUSE(TABLE '02188_test_dictionary_source')) +LAYOUT(DIRECT()); + +SELECT 'Dictionary output'; +SELECT * FROM 02188_test_dictionary_simple_primary_key; +DROP DICTIONARY 02188_test_dictionary_simple_primary_key; + +DROP DICTIONARY IF EXISTS 02188_test_dictionary_complex_primary_key; +CREATE DICTIONARY 02188_test_dictionary_complex_primary_key +( + id UInt64, + value String +) +PRIMARY KEY id, value +SOURCE(CLICKHOUSE(TABLE '02188_test_dictionary_source')) +LAYOUT(COMPLEX_KEY_DIRECT()); + +SELECT 'Dictionary output'; +SELECT * FROM 02188_test_dictionary_complex_primary_key; +DROP DICTIONARY 02188_test_dictionary_complex_primary_key; + +CREATE DICTIONARY 02188_test_dictionary_complex_primary_key +( + id UInt64, + value String +) +PRIMARY KEY (id, value) +SOURCE(CLICKHOUSE(TABLE '02188_test_dictionary_source')) +LAYOUT(COMPLEX_KEY_DIRECT()); + +SELECT 'Dictionary output'; +SELECT * FROM 02188_test_dictionary_complex_primary_key; +DROP DICTIONARY 02188_test_dictionary_complex_primary_key; + +DROP TABLE 02188_test_dictionary_source; diff --git a/tests/queries/0_stateless/02188_table_function_format.reference b/tests/queries/0_stateless/02188_table_function_format.reference new file mode 100644 index 00000000000..ab568fb9fe5 --- /dev/null +++ b/tests/queries/0_stateless/02188_table_function_format.reference @@ -0,0 +1,52 @@ +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +111 Hello +123 World +1 2 [1,2,3] [['abc'],[],['d','e']] +c1 Nullable(Float64) +c2 Nullable(Float64) +c3 Array(Nullable(Float64)) +c4 Array(Array(Nullable(String))) +111 Hello +123 World +111 Hello +131 Hello +123 World +b Nullable(Float64) +a Nullable(String) diff --git a/tests/queries/0_stateless/02188_table_function_format.sql b/tests/queries/0_stateless/02188_table_function_format.sql new file mode 100644 index 00000000000..ff8e2a0d53c --- /dev/null +++ b/tests/queries/0_stateless/02188_table_function_format.sql @@ -0,0 +1,70 @@ +-- Tags: no-fasttest + +select * from format(JSONEachRow, +$$ +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +$$); + +set max_block_size=5; + +select * from format(JSONEachRow, +$$ +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +$$); + +select * from format(CSV, '1,2,"[1,2,3]","[[\'abc\'], [], [\'d\', \'e\']]"'); +desc format(CSV, '1,2,"[1,2,3]","[[\'abc\'], [], [\'d\', \'e\']]"'); + +drop table if exists test; + +create table test as format(JSONEachRow, +$$ +{"a": "Hello", "b": 111} +{"a": "World", "b": 123} +{"a": "Hello", "b": 111} +{"a": "Hello", "b": 131} +{"a": "World", "b": 123} +$$); + +select * from test; +desc table test; +drop table test; + diff --git a/tests/queries/0_stateless/02189_join_type_conversion.reference b/tests/queries/0_stateless/02189_join_type_conversion.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02189_join_type_conversion.sql b/tests/queries/0_stateless/02189_join_type_conversion.sql new file mode 100644 index 00000000000..918065383c8 --- /dev/null +++ b/tests/queries/0_stateless/02189_join_type_conversion.sql @@ -0,0 +1 @@ +SELECT t1.*, t2.* FROM (SELECT 1 AS k) t1 JOIN (SELECT -1 AS k) t2 ON t1.k = t2.k; diff --git a/tests/queries/0_stateless/02190_current_metrics_query.reference b/tests/queries/0_stateless/02190_current_metrics_query.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02190_current_metrics_query.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02190_current_metrics_query.sql b/tests/queries/0_stateless/02190_current_metrics_query.sql new file mode 100644 index 00000000000..e8b22e92a99 --- /dev/null +++ b/tests/queries/0_stateless/02190_current_metrics_query.sql @@ -0,0 +1,2 @@ +-- This query itself is also accounted in metric. +SELECT value > 0 FROM system.metrics WHERE metric = 'Query'; diff --git a/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.reference b/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.reference new file mode 100644 index 00000000000..858b670b9d9 --- /dev/null +++ b/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.reference @@ -0,0 +1,3 @@ +xyz\rabc +Hello, world +End diff --git a/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.sh b/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.sh new file mode 100755 index 00000000000..1f6c040a34b --- /dev/null +++ b/tests/queries/0_stateless/02190_format_regexp_cr_in_the_middle.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, long + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +echo -ne 'xyz\rabc\nHello, world\r\nEnd' | ${CLICKHOUSE_LOCAL} --structure 's String' --input-format Regexp --format_regexp '(.*)' --query 'SELECT * FROM table' diff --git a/tests/queries/0_stateless/02191_nested_with_dots.reference b/tests/queries/0_stateless/02191_nested_with_dots.reference new file mode 100644 index 00000000000..86cee7a8110 --- /dev/null +++ b/tests/queries/0_stateless/02191_nested_with_dots.reference @@ -0,0 +1,7 @@ +[1] [[1]] +[[1]] +[(1,[1])] +[[1]] +(('a',1),'b') +a 1 +a b diff --git a/tests/queries/0_stateless/02191_nested_with_dots.sql b/tests/queries/0_stateless/02191_nested_with_dots.sql new file mode 100644 index 00000000000..cf649ca3013 --- /dev/null +++ b/tests/queries/0_stateless/02191_nested_with_dots.sql @@ -0,0 +1,33 @@ +DROP TABLE IF EXISTS t_nested_with_dots; + +CREATE TABLE t_nested_with_dots (n Nested(id UInt64, `values.id` Array(UInt64))) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_nested_with_dots VALUES ([1], [[1]]); + +SELECT * FROM t_nested_with_dots; +SELECT n.values.id FROM t_nested_with_dots; + +DROP TABLE IF EXISTS t_nested_with_dots; +SET flatten_nested = 0; + +CREATE TABLE t_nested_with_dots (n Nested(id UInt64, `values.id` Array(UInt64))) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_nested_with_dots VALUES ([(1, [1])]); + +SELECT * FROM t_nested_with_dots; +SELECT n.values.id FROM t_nested_with_dots; + +DROP TABLE IF EXISTS t_nested_with_dots; + +CREATE TABLE t_nested_with_dots (`t.t2` Tuple(`t3.t4.t5` Tuple(`s1.s2` String, `u1.u2` UInt64), `s3.s4.s5` String)) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_nested_with_dots VALUES ((('a', 1), 'b')); + +SELECT * FROM t_nested_with_dots; +SELECT t.t2.t3.t4.t5.s1.s2, t.t2.t3.t4.t5.u1.u2 FROM t_nested_with_dots; +SELECT t.t2.t3.t4.t5.s1.s2, t.t2.s3.s4.s5 FROM t_nested_with_dots; + +DROP TABLE IF EXISTS t_nested_with_dots; diff --git a/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.reference b/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.reference new file mode 100644 index 00000000000..227e3b013b2 --- /dev/null +++ b/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.reference @@ -0,0 +1,10 @@ +2022-01-01 01:02:03 +2022-01-01 01:02:03 +2022-01-01 01:02:03 +2022-01-01 01:02:03 +2022-01-01 01:02:00 +2022-01-01 01:02:00 +2021-12-31 22:58:00 +2022-01-01 02:02:03 +2022-01-01 00:02:03 +2022-01-01 02:02:03 diff --git a/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.sql b/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.sql new file mode 100644 index 00000000000..d30834b90a3 --- /dev/null +++ b/tests/queries/0_stateless/02191_parse_date_time_best_effort_more_cases.sql @@ -0,0 +1,10 @@ +SELECT parseDateTimeBestEffort('20220101-010203', 'UTC'); +SELECT parseDateTimeBestEffort('20220101+010203', 'UTC'); +SELECT parseDateTimeBestEffort('20220101 010203', 'UTC'); +SELECT parseDateTimeBestEffort('20220101T010203', 'UTC'); +SELECT parseDateTimeBestEffort('20220101T01:02', 'UTC'); +SELECT parseDateTimeBestEffort('20220101-0102', 'UTC'); +SELECT parseDateTimeBestEffort('20220101+0102', 'UTC'); +SELECT parseDateTimeBestEffort('20220101-010203-01', 'UTC'); +SELECT parseDateTimeBestEffort('20220101-010203+0100', 'UTC'); +SELECT parseDateTimeBestEffort('20220101-010203-01:00', 'UTC'); diff --git a/tests/queries/0_stateless/02192_comment.reference b/tests/queries/0_stateless/02192_comment.reference new file mode 100644 index 00000000000..700b3808344 --- /dev/null +++ b/tests/queries/0_stateless/02192_comment.reference @@ -0,0 +1,7 @@ +1 +2 +3 +1 +1 +1 +\n# hello 1 diff --git a/tests/queries/0_stateless/02192_comment.sql b/tests/queries/0_stateless/02192_comment.sql new file mode 100644 index 00000000000..ff56caa77ee --- /dev/null +++ b/tests/queries/0_stateless/02192_comment.sql @@ -0,0 +1,16 @@ +# comment +#! comment2 +select 1; +# comment3 +#! comment4 +select 2; # another one comemnt +# +#! +select 3; + +select 1; #! +SELECT # hello +1; +SELECT /* # hello */ 1; +SELECT ' +# hello', 1; diff --git a/tests/queries/0_stateless/02192_comment_error.reference b/tests/queries/0_stateless/02192_comment_error.reference new file mode 100644 index 00000000000..21da4d2be9e --- /dev/null +++ b/tests/queries/0_stateless/02192_comment_error.reference @@ -0,0 +1,5 @@ +OK +OK +OK +OK +OK diff --git a/tests/queries/0_stateless/02192_comment_error.sh b/tests/queries/0_stateless/02192_comment_error.sh new file mode 100755 index 00000000000..78ff474ae84 --- /dev/null +++ b/tests/queries/0_stateless/02192_comment_error.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} --query="#" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CLIENT} --query="#not a comemnt" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CLIENT} --query="select 1 #not a comemnt" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CLIENT} --query="select 1 #" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "select 42 #" 2>&1 | grep -F -q 'Syntax error' && echo 'OK' || echo 'FAIL' diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_1.reference b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.reference new file mode 100644 index 00000000000..333b3b69fb6 --- /dev/null +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.reference @@ -0,0 +1,5 @@ +1 aaa +2 bbb +3 ccc +4 ddd +4 4 diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql new file mode 100644 index 00000000000..795a27883e6 --- /dev/null +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_1.sql @@ -0,0 +1,27 @@ +DROP TABLE IF EXISTS t_async_insert_02193_1; + +CREATE TABLE t_async_insert_02193_1 (id UInt32, s String) ENGINE = Memory; + +INSERT INTO t_async_insert_02193_1 FORMAT CSV SETTINGS async_insert = 1 +1,aaa +; + +INSERT INTO t_async_insert_02193_1 FORMAT Values SETTINGS async_insert = 1 (2, 'bbb'); + +SET async_insert = 1; + +INSERT INTO t_async_insert_02193_1 VALUES (3, 'ccc'); +INSERT INTO t_async_insert_02193_1 FORMAT JSONEachRow {"id": 4, "s": "ddd"}; + +SELECT * FROM t_async_insert_02193_1 ORDER BY id; + +SYSTEM FLUSH LOGS; + +SELECT count(), sum(ProfileEvents['AsyncInsertQuery']) FROM system.query_log +WHERE + event_date >= yesterday() AND + type = 'QueryFinish' AND + current_database = currentDatabase() AND + query ILIKE 'INSERT INTO t_async_insert_02193_1%'; + +DROP TABLE IF EXISTS t_async_insert_02193_1; diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_2.reference b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.reference new file mode 100644 index 00000000000..398d1450c6c --- /dev/null +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.reference @@ -0,0 +1,4 @@ +1 aaa +2 bbb +3 ccc +4 ddd diff --git a/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh new file mode 100755 index 00000000000..e620b21ac72 --- /dev/null +++ b/tests/queries/0_stateless/02193_async_insert_tcp_client_2.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: long + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_async_insert_02193_2" + +${CLICKHOUSE_CLIENT} -q "CREATE TABLE t_async_insert_02193_2 (id UInt32, s String) ENGINE = Memory" + +${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 FORMAT CSV SETTINGS async_insert = 1 1,aaa" +${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 FORMAT Values SETTINGS async_insert = 1 (2, 'bbb')" + +${CLICKHOUSE_CLIENT} -q "INSERT INTO t_async_insert_02193_2 VALUES (3, 'ccc')" --async_insert=1 +${CLICKHOUSE_CLIENT} -q 'INSERT INTO t_async_insert_02193_2 FORMAT JSONEachRow {"id": 4, "s": "ddd"}' --async_insert=1 + +${CLICKHOUSE_CLIENT} -q "SELECT * FROM t_async_insert_02193_2 ORDER BY id" +${CLICKHOUSE_CLIENT} -q "TRUNCATE TABLE t_async_insert_02193_2" + +${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS t_async_insert_02193_2" diff --git a/tests/queries/0_stateless/02200_use_skip_indexes.reference b/tests/queries/0_stateless/02200_use_skip_indexes.reference new file mode 100644 index 00000000000..f11db7af711 --- /dev/null +++ b/tests/queries/0_stateless/02200_use_skip_indexes.reference @@ -0,0 +1,4 @@ +-- { echoOn } +SELECT * FROM data_02200 WHERE value = 1 SETTINGS use_skip_indexes=1, max_rows_to_read=1; +1 1 +SELECT * FROM data_02200 WHERE value = 1 SETTINGS use_skip_indexes=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } diff --git a/tests/queries/0_stateless/02200_use_skip_indexes.sql b/tests/queries/0_stateless/02200_use_skip_indexes.sql new file mode 100644 index 00000000000..64003285abb --- /dev/null +++ b/tests/queries/0_stateless/02200_use_skip_indexes.sql @@ -0,0 +1,14 @@ +CREATE TABLE data_02200 ( + key Int, + value Int, + INDEX idx value TYPE minmax GRANULARITY 1 +) +Engine=MergeTree() +ORDER BY key +PARTITION BY key; + +INSERT INTO data_02200 SELECT number, number FROM numbers(10); + +-- { echoOn } +SELECT * FROM data_02200 WHERE value = 1 SETTINGS use_skip_indexes=1, max_rows_to_read=1; +SELECT * FROM data_02200 WHERE value = 1 SETTINGS use_skip_indexes=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } diff --git a/tests/queries/0_stateless/02201_use_skip_indexes_if_final.reference b/tests/queries/0_stateless/02201_use_skip_indexes_if_final.reference new file mode 100644 index 00000000000..423a9fa46d6 --- /dev/null +++ b/tests/queries/0_stateless/02201_use_skip_indexes_if_final.reference @@ -0,0 +1,6 @@ +-- { echoOn } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=0, use_skip_indexes_if_final=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=0, use_skip_indexes_if_final=1, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=1, max_rows_to_read=1; +1 1 diff --git a/tests/queries/0_stateless/02201_use_skip_indexes_if_final.sql b/tests/queries/0_stateless/02201_use_skip_indexes_if_final.sql new file mode 100644 index 00000000000..2afc4941c9e --- /dev/null +++ b/tests/queries/0_stateless/02201_use_skip_indexes_if_final.sql @@ -0,0 +1,16 @@ +CREATE TABLE data_02201 ( + key Int, + value Int, + INDEX idx value TYPE minmax GRANULARITY 1 +) +Engine=AggregatingMergeTree() +ORDER BY key +PARTITION BY key; + +INSERT INTO data_02201 SELECT number, number FROM numbers(10); + +-- { echoOn } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=0, use_skip_indexes_if_final=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=0, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=0, use_skip_indexes_if_final=1, max_rows_to_read=1; -- { serverError TOO_MANY_ROWS } +SELECT * FROM data_02201 FINAL WHERE value = 1 SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=1, max_rows_to_read=1; diff --git a/tests/queries/0_stateless/02202_use_skip_indexes_if_final.reference b/tests/queries/0_stateless/02202_use_skip_indexes_if_final.reference new file mode 100644 index 00000000000..7d543cfcaf6 --- /dev/null +++ b/tests/queries/0_stateless/02202_use_skip_indexes_if_final.reference @@ -0,0 +1,6 @@ +-- { echoOn } +SELECT * FROM data_02201 FINAL WHERE value_max = 1 ORDER BY key, value_max SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=0; +0 1 +SELECT * FROM data_02201 FINAL WHERE value_max = 1 ORDER BY key, value_max SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=1; +0 1 +1 1 diff --git a/tests/queries/0_stateless/02202_use_skip_indexes_if_final.sql b/tests/queries/0_stateless/02202_use_skip_indexes_if_final.sql new file mode 100644 index 00000000000..cce785eb17d --- /dev/null +++ b/tests/queries/0_stateless/02202_use_skip_indexes_if_final.sql @@ -0,0 +1,19 @@ +-- This tests will show the difference in data with use_skip_indexes_if_final and w/o + +CREATE TABLE data_02201 ( + key Int, + value_max SimpleAggregateFunction(max, Int), + INDEX idx value_max TYPE minmax GRANULARITY 1 +) +Engine=AggregatingMergeTree() +ORDER BY key +PARTITION BY key; + +SYSTEM STOP MERGES data_02201; + +INSERT INTO data_02201 SELECT number, number FROM numbers(10); +INSERT INTO data_02201 SELECT number, number+1 FROM numbers(10); + +-- { echoOn } +SELECT * FROM data_02201 FINAL WHERE value_max = 1 ORDER BY key, value_max SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=0; +SELECT * FROM data_02201 FINAL WHERE value_max = 1 ORDER BY key, value_max SETTINGS use_skip_indexes=1, use_skip_indexes_if_final=1; diff --git a/tests/queries/0_stateless/02203_shebang b/tests/queries/0_stateless/02203_shebang new file mode 100755 index 00000000000..07686d1aab4 --- /dev/null +++ b/tests/queries/0_stateless/02203_shebang @@ -0,0 +1,3 @@ +#!/usr/bin/clickhouse-local --queries-file + +SELECT 1; diff --git a/tests/queries/0_stateless/02203_shebang.reference b/tests/queries/0_stateless/02203_shebang.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02203_shebang.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02203_shebang.sh b/tests/queries/0_stateless/02203_shebang.sh new file mode 100755 index 00000000000..6c46a15d4f5 --- /dev/null +++ b/tests/queries/0_stateless/02203_shebang.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +sed -i.bak "s!/usr/bin/clickhouse-local!$(command -v ${CLICKHOUSE_LOCAL})!" "${CUR_DIR}/02203_shebang" +"${CUR_DIR}/02203_shebang" diff --git a/tests/queries/0_stateless/02204_fractional_progress_bar_long.reference b/tests/queries/0_stateless/02204_fractional_progress_bar_long.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02204_fractional_progress_bar_long.sh b/tests/queries/0_stateless/02204_fractional_progress_bar_long.sh new file mode 100755 index 00000000000..aa8aa22be5c --- /dev/null +++ b/tests/queries/0_stateless/02204_fractional_progress_bar_long.sh @@ -0,0 +1,20 @@ +#!/usr/bin/expect -f +# Tags: no-tsan, no-asan, no-ubsan, no-msan, no-debug, no-fasttest + +log_user 0 +set timeout 60 +match_max 100000 + +spawn clickhouse-local --progress --query "SELECT sum(number % 100000000 = 12345678 ? sleep(0.1) : 1) FROM numbers(1000000000)" + +expect { + "▏" { exit 0 } + "▎" { exit 0 } + "▍" { exit 0 } + "▌" { exit 0 } + "▋" { exit 0 } + "▋" { exit 0 } + "▊" { exit 0 } + "▉" { exit 0 } + timeout { exit 1 } +} diff --git a/tests/queries/0_stateless/02205_HTTP_user_agent.python b/tests/queries/0_stateless/02205_HTTP_user_agent.python new file mode 100644 index 00000000000..8fb9cea0845 --- /dev/null +++ b/tests/queries/0_stateless/02205_HTTP_user_agent.python @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 + +from http.server import SimpleHTTPRequestHandler,HTTPServer +import socket +import sys +import threading +import os +import traceback +import urllib.request +import subprocess + + +def is_ipv6(host): + try: + socket.inet_aton(host) + return False + except: + return True + +def get_local_port(host, ipv6): + if ipv6: + family = socket.AF_INET6 + else: + family = socket.AF_INET + + with socket.socket(family) as fd: + fd.bind((host, 0)) + return fd.getsockname()[1] + +CLICKHOUSE_HOST = os.environ.get('CLICKHOUSE_HOST', 'localhost') +CLICKHOUSE_PORT_HTTP = os.environ.get('CLICKHOUSE_PORT_HTTP', '8123') + +# Server returns this JSON response. +SERVER_JSON_RESPONSE = \ +'''{ + "login": "ClickHouse", + "id": 54801242, + "name": "ClickHouse", + "company": null +}''' + +EXPECTED_ANSWER = \ +'''{\\n\\t"login": "ClickHouse",\\n\\t"id": 54801242,\\n\\t"name": "ClickHouse",\\n\\t"company": null\\n}''' + +##################################################################################### +# This test starts an HTTP server and serves data to clickhouse url-engine based table. +# The objective of this test is to check the ClickHouse server provides a User-Agent +# with HTTP requests. +# In order for it to work ip+port of http server (given below) should be +# accessible from clickhouse server. +##################################################################################### + +# IP-address of this host accessible from the outside world. Get the first one +HTTP_SERVER_HOST = subprocess.check_output(['hostname', '-i']).decode('utf-8').strip().split()[0] +IS_IPV6 = is_ipv6(HTTP_SERVER_HOST) +HTTP_SERVER_PORT = get_local_port(HTTP_SERVER_HOST, IS_IPV6) + +# IP address and port of the HTTP server started from this script. +HTTP_SERVER_ADDRESS = (HTTP_SERVER_HOST, HTTP_SERVER_PORT) +if IS_IPV6: + HTTP_SERVER_URL_STR = 'http://' + f'[{str(HTTP_SERVER_ADDRESS[0])}]:{str(HTTP_SERVER_ADDRESS[1])}' + "/" +else: + HTTP_SERVER_URL_STR = 'http://' + f'{str(HTTP_SERVER_ADDRESS[0])}:{str(HTTP_SERVER_ADDRESS[1])}' + "/" + + +def get_ch_answer(query): + host = CLICKHOUSE_HOST + if IS_IPV6: + host = f'[{host}]' + + url = os.environ.get('CLICKHOUSE_URL', 'http://{host}:{port}'.format(host=CLICKHOUSE_HOST, port=CLICKHOUSE_PORT_HTTP)) + return urllib.request.urlopen(url, data=query.encode()).read().decode() + +def check_answers(query, answer): + ch_answer = get_ch_answer(query) + if ch_answer.strip() != answer.strip(): + print("FAIL on query:", query, file=sys.stderr) + print("Expected answer:", answer, file=sys.stderr) + print("Fetched answer :", ch_answer, file=sys.stderr) + raise Exception("Fail on query") + +# Server with check for User-Agent headers. +class HttpProcessor(SimpleHTTPRequestHandler): + def _set_headers(self): + user_agent = self.headers.get('User-Agent') + if user_agent and user_agent.startswith('ClickHouse/'): + self.send_response(200) + else: + self.send_response(403) + + self.send_header('Content-Type', 'text/csv') + self.end_headers() + + def do_GET(self): + self._set_headers() + self.wfile.write(SERVER_JSON_RESPONSE.encode()) + + def log_message(self, format, *args): + return + +class HTTPServerV6(HTTPServer): + address_family = socket.AF_INET6 + +def start_server(requests_amount): + if IS_IPV6: + httpd = HTTPServerV6(HTTP_SERVER_ADDRESS, HttpProcessor) + else: + httpd = HTTPServer(HTTP_SERVER_ADDRESS, HttpProcessor) + + def real_func(): + for i in range(requests_amount): + httpd.handle_request() + + t = threading.Thread(target=real_func) + return t + +##################################################################### +# Testing area. +##################################################################### + +def test_select(): + global HTTP_SERVER_URL_STR + query = 'SELECT * FROM url(\'{}\',\'JSONAsString\');'.format(HTTP_SERVER_URL_STR) + check_answers(query, EXPECTED_ANSWER) + +def main(): + t = start_server(1) + t.start() + test_select() + t.join() + print("PASSED") + +if __name__ == "__main__": + try: + main() + except Exception as ex: + exc_type, exc_value, exc_traceback = sys.exc_info() + traceback.print_tb(exc_traceback, file=sys.stderr) + print(ex, file=sys.stderr) + sys.stderr.flush() + + os._exit(1) + diff --git a/tests/queries/0_stateless/02205_HTTP_user_agent.reference b/tests/queries/0_stateless/02205_HTTP_user_agent.reference new file mode 100644 index 00000000000..53cdf1e9393 --- /dev/null +++ b/tests/queries/0_stateless/02205_HTTP_user_agent.reference @@ -0,0 +1 @@ +PASSED diff --git a/tests/queries/0_stateless/02205_HTTP_user_agent.sh b/tests/queries/0_stateless/02205_HTTP_user_agent.sh new file mode 100755 index 00000000000..b125e91ae85 --- /dev/null +++ b/tests/queries/0_stateless/02205_HTTP_user_agent.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +python3 "$CURDIR"/02205_HTTP_user_agent.python + diff --git a/tests/queries/0_stateless/02205_map_populate_series_non_const.reference b/tests/queries/0_stateless/02205_map_populate_series_non_const.reference new file mode 100644 index 00000000000..5d938d2917d --- /dev/null +++ b/tests/queries/0_stateless/02205_map_populate_series_non_const.reference @@ -0,0 +1,34 @@ +mapPopulateSeries with map +Without max key +{0:5} +{0:5,1:0,2:0,3:0,4:0,5:10} +{-5:-5,-4:0,-3:0,-2:0,-1:0,0:5,1:0,2:0,3:0,4:0,5:10} +{-5:-5,-4:0,-3:0,-2:0,-1:0,0:5,1:0,2:0,3:0,4:0,5:10,6:0,7:0,8:0,9:0,10:15} +With max key +{0:5,1:0,2:0,3:0,4:0,5:0,6:0,7:0,8:0,9:0,10:0,11:0,12:0,13:0,14:0,15:0,16:0,17:0,18:0,19:0,20:0} +{0:5,1:0,2:0,3:0,4:0,5:10,6:0,7:0,8:0,9:0,10:0,11:0,12:0,13:0,14:0,15:0,16:0,17:0,18:0,19:0,20:0} +{-5:-5,-4:0,-3:0,-2:0,-1:0,0:5,1:0,2:0,3:0,4:0,5:10} +{-5:-5,-4:0,-3:0,-2:0,-1:0,0:5,1:0,2:0,3:0,4:0,5:10,6:0,7:0,8:0,9:0,10:15,11:0,12:0,13:0,14:0,15:0,16:0,17:0,18:0,19:0,20:0} +Possible verflow +{18446744073709551610:5,18446744073709551611:0,18446744073709551612:0,18446744073709551613:0,18446744073709551614:0,18446744073709551615:0} +{18446744073709551615:5} +Duplicate keys +{1:4,2:0,3:0,4:0,5:6} +{1:4,2:0,3:0,4:0,5:6,6:0,7:0,8:0,9:0,10:0} +mapPopulateSeries with two arrays +Without max key +([0],[5]) +([0,1,2,3,4,5],[5,0,0,0,0,10]) +([-5,-4,-3,-2,-1,0,1,2,3,4,5],[-5,0,0,0,0,5,0,0,0,0,10]) +([-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10],[-5,0,0,0,0,5,0,0,0,0,10,0,0,0,0,15]) +With max key +([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],[5,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]) +([0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],[5,0,0,0,0,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]) +([-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],[-5,0,0,0,0,5,0,0,0,0,10,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]) +([-5,-4,-3,-2,-1,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20],[-5,0,0,0,0,5,0,0,0,0,10,0,0,0,0,15,0,0,0,0,0,0,0,0,0,0]) +Possible verflow +([18446744073709551610,18446744073709551611,18446744073709551612,18446744073709551613,18446744073709551614,18446744073709551615],[5,0,0,0,0,0]) +([18446744073709551615],[5]) +Duplicate keys +([1,2,3,4,5],[4,0,0,0,6]) +([1,2,3,4,5,6,7,8,9,10],[4,0,0,0,6,0,0,0,0,0]) diff --git a/tests/queries/0_stateless/02205_map_populate_series_non_const.sql b/tests/queries/0_stateless/02205_map_populate_series_non_const.sql new file mode 100644 index 00000000000..08a3dd51eb1 --- /dev/null +++ b/tests/queries/0_stateless/02205_map_populate_series_non_const.sql @@ -0,0 +1,125 @@ +DROP TABLE IF EXISTS 02005_test_table; +CREATE TABLE 02005_test_table +( + value Map(Int64, Int64) +) +ENGINE = TinyLog; + +SELECT 'mapPopulateSeries with map'; + +SELECT 'Without max key'; + +SELECT mapPopulateSeries(value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES (map(0, 5)); +SELECT mapPopulateSeries(value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES (map(0, 5, 5, 10)); +SELECT mapPopulateSeries(value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES (map(-5, -5, 0, 5, 5, 10)); +SELECT mapPopulateSeries(value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES (map(-5, -5, 0, 5, 5, 10, 10, 15)); +SELECT mapPopulateSeries(value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +SELECT 'With max key'; + +SELECT mapPopulateSeries(value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES (map(0, 5)); +SELECT mapPopulateSeries(value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES (map(0, 5, 5, 10)); +SELECT mapPopulateSeries(value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES (map(-5, -5, 0, 5, 5, 10)); +SELECT mapPopulateSeries(value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES (map(-5, -5, 0, 5, 5, 10, 10, 15)); +SELECT mapPopulateSeries(value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +SELECT 'Possible verflow'; + +SELECT mapPopulateSeries(map(toUInt64(18446744073709551610), toUInt64(5)), 18446744073709551615); +SELECT mapPopulateSeries(map(toUInt64(18446744073709551615), toUInt64(5)), 18446744073709551615); + +SELECT 'Duplicate keys'; + +SELECT mapPopulateSeries(map(1, 4, 1, 5, 5, 6)); +SELECT mapPopulateSeries(map(1, 4, 1, 5, 5, 6), materialize(10)); + +DROP TABLE 02005_test_table; + +DROP TABLE IF EXISTS 02005_test_table; +CREATE TABLE 02005_test_table +( + key Array(Int64), + value Array(Int64) +) +ENGINE = TinyLog; + +SELECT 'mapPopulateSeries with two arrays'; +SELECT 'Without max key'; + +SELECT mapPopulateSeries(key, value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES ([0], [5]); +SELECT mapPopulateSeries(key, value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES ([0, 5], [5, 10]); +SELECT mapPopulateSeries(key, value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES ([-5, 0, 5], [-5, 5, 10]); +SELECT mapPopulateSeries(key, value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES ([-5, 0, 5, 10], [-5, 5, 10, 15]); +SELECT mapPopulateSeries(key, value) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +SELECT 'With max key'; + +SELECT mapPopulateSeries(key, value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES ([0], [5]); +SELECT mapPopulateSeries(key, value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES ([0, 5], [5, 10]); +SELECT mapPopulateSeries(key, value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES ([-5, 0, 5], [-5, 5, 10]); +SELECT mapPopulateSeries(key, value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +INSERT INTO 02005_test_table VALUES ([-5, 0, 5, 10], [-5, 5, 10, 15]); +SELECT mapPopulateSeries(key, value, materialize(20)) FROM 02005_test_table; +TRUNCATE TABLE 02005_test_table; + +SELECT 'Possible verflow'; + +SELECT mapPopulateSeries([18446744073709551610], [5], 18446744073709551615); +SELECT mapPopulateSeries([18446744073709551615], [5], 18446744073709551615); + +SELECT 'Duplicate keys'; + +SELECT mapPopulateSeries([1, 1, 5], [4, 5, 6]); +SELECT mapPopulateSeries([1, 1, 5], [4, 5, 6], materialize(10)); + +DROP TABLE 02005_test_table; diff --git a/tests/queries/0_stateless/02205_postgresql_functions.reference b/tests/queries/0_stateless/02205_postgresql_functions.reference new file mode 100644 index 00000000000..7da18b92001 --- /dev/null +++ b/tests/queries/0_stateless/02205_postgresql_functions.reference @@ -0,0 +1,100 @@ +1 1 +1 1 +1 1 +1 1 +. . . 1 +. . . 1 +. . . 1 +._ .o .o 1 +. . . 1 +_. o. o. 1 +. . . 1 +_._ o.o o.o 1 +._ .o .o 1 +. . . 1 +._ .o .o 1 +._ .o .o 1 +._ .o .o 1 +_. o. o. 1 +._ .o .o 1 +_._ o.o o.o 1 +_. o. o. 1 +. . . 1 +_. o. o. 1 +._ .o .o 1 +_. o. o. 1 +_. o. o. 1 +_. o. o. 1 +_._ o.o o.o 1 +_._ o.o o.o 1 +. . . 1 +_._ o.o o.o 1 +._ .o .o 1 +_._ o.o o.o 1 +_. o. o. 1 +_._ o.o o.o 1 +_._ o.o o.o 1 +. . . 1 +. . . 1 +. . . 1 +._ .oo .oo 1 +. . . 1 +_. oo. oo. 1 +. . . 1 +_._ oo.oo oo.oo 1 +._ .oo .oo 1 +. . . 1 +._ .oo .oo 1 +._ .oo .oo 1 +._ .oo .oo 1 +_. oo. oo. 1 +._ .oo .oo 1 +_._ oo.oo oo.oo 1 +_. oo. oo. 1 +. . . 1 +_. oo. oo. 1 +._ .oo .oo 1 +_. oo. oo. 1 +_. oo. oo. 1 +_. oo. oo. 1 +_._ oo.oo oo.oo 1 +_._ oo.oo oo.oo 1 +. . . 1 +_._ oo.oo oo.oo 1 +._ .oo .oo 1 +_._ oo.oo oo.oo 1 +_. oo. oo. 1 +_._ oo.oo oo.oo 1 +_._ oo.oo oo.oo 1 +. . . 1 +. . . 1 +. . . 1 +.__ .oo .oo 1 +. . . 1 +__. oo. oo. 1 +. . . 1 +__.__ oo.oo oo.oo 1 +.__ .oo .oo 1 +. . . 1 +.__ .oo .oo 1 +.__ .oo .oo 1 +.__ .oo .oo 1 +__. oo. oo. 1 +.__ .oo .oo 1 +__.__ oo.oo oo.oo 1 +__. oo. oo. 1 +. . . 1 +__. oo. oo. 1 +.__ .oo .oo 1 +__. oo. oo. 1 +__. oo. oo. 1 +__. oo. oo. 1 +__.__ oo.oo oo.oo 1 +__.__ oo.oo oo.oo 1 +. . . 1 +__.__ oo.oo oo.oo 1 +.__ .oo .oo 1 +__.__ oo.oo oo.oo 1 +__. oo. oo. 1 +__.__ oo.oo oo.oo 1 +__.__ oo.oo oo.oo 1 diff --git a/tests/queries/0_stateless/02205_postgresql_functions.sql b/tests/queries/0_stateless/02205_postgresql_functions.sql new file mode 100644 index 00000000000..343149f5272 --- /dev/null +++ b/tests/queries/0_stateless/02205_postgresql_functions.sql @@ -0,0 +1,59 @@ +--- REGEXP_MATCHES +select match('a key="v" ', 'key="(.*?)"'), REGEXP_MATCHES('a key="v" ', 'key="(.*?)"'); +select match(materialize('a key="v" '), 'key="(.*?)"'), REGEXP_MATCHES(materialize('a key="v" '), 'key="(.*?)"'); + +select match('\0 key="v" ', 'key="(.*?)"'), REGEXP_MATCHES('\0 key="v" ', 'key="(.*?)"'); +select match(materialize('\0 key="v" '), 'key="(.*?)"'), REGEXP_MATCHES(materialize('\0 key="v" '), 'key="(.*?)"'); + + +--- REGEXP_REPLACE +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.', '.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.', '._']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.', '_.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.', '_._']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['._', '.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['._', '._']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['._', '_.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['._', '_._']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['_.', '.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['_.', '._']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['_.', '_.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['_.', '_._']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['_._', '.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['_._', '._']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['_._', '_.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['_._', '_._']) AS s); + +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['.', '.']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['.', '._']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['.', '_.']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['.', '_._']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['._', '.']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['._', '._']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['._', '_.']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['._', '_._']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['_.', '.']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['_.', '._']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['_.', '_.']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['_.', '_._']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['_._', '.']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['_._', '._']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['_._', '_.']) AS s); +SELECT s, replaceAll(s, '_', 'oo') AS a, REGEXP_REPLACE(s, '_', 'oo') AS b, a = b FROM (SELECT arrayJoin(['_._', '_._']) AS s); + +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.', '.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.', '.__']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.', '__.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.', '__.__']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.__', '.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.__', '.__']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.__', '__.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['.__', '__.__']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.', '.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.', '.__']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.', '__.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.', '__.__']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.__', '.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.__', '.__']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.__', '__.']) AS s); +SELECT s, replaceAll(s, '_', 'o') AS a, REGEXP_REPLACE(s, '_', 'o') AS b, a = b FROM (SELECT arrayJoin(['__.__', '__.__']) AS s); diff --git a/tests/queries/0_stateless/02206_clickhouse_local_use_database.reference b/tests/queries/0_stateless/02206_clickhouse_local_use_database.reference new file mode 100644 index 00000000000..679b6811f8e --- /dev/null +++ b/tests/queries/0_stateless/02206_clickhouse_local_use_database.reference @@ -0,0 +1,12 @@ +SHOW TABLES; +CREATE DATABASE test1; +CREATE TABLE test1.table1 (a Int32) ENGINE=Memory; +USE test1; +SHOW TABLES; +table1 +CREATE DATABASE test2; +USE test2; +SHOW TABLES; +USE test1; +SHOW TABLES; +table1 diff --git a/tests/queries/0_stateless/02206_clickhouse_local_use_database.sh b/tests/queries/0_stateless/02206_clickhouse_local_use_database.sh new file mode 100755 index 00000000000..59ede739e4a --- /dev/null +++ b/tests/queries/0_stateless/02206_clickhouse_local_use_database.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_LOCAL --echo --multiline --multiquery -q """ +SHOW TABLES; +CREATE DATABASE test1; +CREATE TABLE test1.table1 (a Int32) ENGINE=Memory; +USE test1; +SHOW TABLES; +CREATE DATABASE test2; +USE test2; +SHOW TABLES; +USE test1; +SHOW TABLES; +""" diff --git a/tests/queries/0_stateless/02206_format_override.reference b/tests/queries/0_stateless/02206_format_override.reference new file mode 100644 index 00000000000..e1bb01eeb2f --- /dev/null +++ b/tests/queries/0_stateless/02206_format_override.reference @@ -0,0 +1,33 @@ +File generated: +Options: --input-format=CSV --output-format JSONEachRow --format TSV +{"num1":"0","num2":"0"} +{"num1":"1","num2":"2"} +{"num1":"2","num2":"4"} +{"num1":"3","num2":"6"} +{"num1":"4","num2":"8"} +{"num1":"5","num2":"10"} +{"num1":"6","num2":"12"} +Options: --input-format=CSV --format TSV +0 0 +1 2 +2 4 +3 6 +4 8 +5 10 +6 12 +Options: --output-format=JSONEachRow --format CSV +{"num1":"0","num2":"0"} +{"num1":"1","num2":"2"} +{"num1":"2","num2":"4"} +{"num1":"3","num2":"6"} +{"num1":"4","num2":"8"} +{"num1":"5","num2":"10"} +{"num1":"6","num2":"12"} +Options: --format CSV +0,0 +1,2 +2,4 +3,6 +4,8 +5,10 +6,12 diff --git a/tests/queries/0_stateless/02206_format_override.sh b/tests/queries/0_stateless/02206_format_override.sh new file mode 100755 index 00000000000..1359f1edeb8 --- /dev/null +++ b/tests/queries/0_stateless/02206_format_override.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +SAMPLE_FILE="$CURDIR/02206_sample_data.csv" + +echo 'File generated:' +${CLICKHOUSE_LOCAL} -q "SELECT number, number * 2 from numbers(7) FORMAT TSV" | tr '\t' ',' >"$SAMPLE_FILE" + + +echo "Options: --input-format=CSV --output-format JSONEachRow --format TSV" +cat "$SAMPLE_FILE" | ${CLICKHOUSE_LOCAL} --input-format CSV --output-format JSONEachRow --format TSV --structure='num1 Int64, num2 Int64' --query='SELECT * from table' + +echo "Options: --input-format=CSV --format TSV" +cat "$SAMPLE_FILE" | ${CLICKHOUSE_LOCAL} --input-format CSV --format TSV --structure='num1 Int64, num2 Int64' --query='SELECT * from table' + +echo "Options: --output-format=JSONEachRow --format CSV" +cat "$SAMPLE_FILE" | ${CLICKHOUSE_LOCAL} --output-format JSONEachRow --format CSV --structure='num1 Int64, num2 Int64' --query='SELECT * from table' + +echo "Options: --format CSV" +cat "$SAMPLE_FILE" | ${CLICKHOUSE_LOCAL} --format CSV --structure='num1 Int64, num2 Int64' --query='SELECT * from table' + +rm "$SAMPLE_FILE" \ No newline at end of file diff --git a/tests/queries/0_stateless/02206_information_schema_show_database.reference b/tests/queries/0_stateless/02206_information_schema_show_database.reference new file mode 100644 index 00000000000..551186fa0ab --- /dev/null +++ b/tests/queries/0_stateless/02206_information_schema_show_database.reference @@ -0,0 +1 @@ +CREATE DATABASE INFORMATION_SCHEMA\nENGINE = Memory diff --git a/tests/queries/0_stateless/02206_information_schema_show_database.sql b/tests/queries/0_stateless/02206_information_schema_show_database.sql new file mode 100644 index 00000000000..de5ca495e2e --- /dev/null +++ b/tests/queries/0_stateless/02206_information_schema_show_database.sql @@ -0,0 +1 @@ +SHOW CREATE DATABASE INFORMATION_SCHEMA; diff --git a/tests/queries/0_stateless/02207_key_condition_floats.reference b/tests/queries/0_stateless/02207_key_condition_floats.reference new file mode 100644 index 00000000000..6c78023f8c0 --- /dev/null +++ b/tests/queries/0_stateless/02207_key_condition_floats.reference @@ -0,0 +1,9 @@ +2 +2 +2 +2 +2 +2 +2 +2 +1 diff --git a/tests/queries/0_stateless/02207_key_condition_floats.sql b/tests/queries/0_stateless/02207_key_condition_floats.sql new file mode 100644 index 00000000000..65527c65290 --- /dev/null +++ b/tests/queries/0_stateless/02207_key_condition_floats.sql @@ -0,0 +1,34 @@ +DROP TABLE IF EXISTS t_key_condition_float; + +CREATE TABLE t_key_condition_float (a Float32) +ENGINE = MergeTree ORDER BY a; + +INSERT INTO t_key_condition_float VALUES (0.1), (0.2); + +SELECT count() FROM t_key_condition_float WHERE a > 0; +SELECT count() FROM t_key_condition_float WHERE a > 0.0; +SELECT count() FROM t_key_condition_float WHERE a > 0::Float32; +SELECT count() FROM t_key_condition_float WHERE a > 0::Float64; + +DROP TABLE t_key_condition_float; + +CREATE TABLE t_key_condition_float (a Float64) +ENGINE = MergeTree ORDER BY a; + +INSERT INTO t_key_condition_float VALUES (0.1), (0.2); + +SELECT count() FROM t_key_condition_float WHERE a > 0; +SELECT count() FROM t_key_condition_float WHERE a > 0.0; +SELECT count() FROM t_key_condition_float WHERE a > 0::Float32; +SELECT count() FROM t_key_condition_float WHERE a > 0::Float64; + +DROP TABLE t_key_condition_float; + +CREATE TABLE t_key_condition_float (a UInt64) +ENGINE = MergeTree ORDER BY a; + +INSERT INTO t_key_condition_float VALUES (1), (2); + +SELECT count() FROM t_key_condition_float WHERE a > 1.5; + +DROP TABLE t_key_condition_float; diff --git a/tests/queries/0_stateless/02207_ttl_move_if_exists.reference b/tests/queries/0_stateless/02207_ttl_move_if_exists.reference new file mode 100644 index 00000000000..bedef1a5ceb --- /dev/null +++ b/tests/queries/0_stateless/02207_ttl_move_if_exists.reference @@ -0,0 +1 @@ +CREATE TABLE default.t_ttl_move_if_exists\n(\n `d` DateTime,\n `a` UInt32\n)\nENGINE = MergeTree\nORDER BY tuple()\nTTL d TO DISK IF EXISTS \'non_existing_disk\'\nSETTINGS index_granularity = 8192 diff --git a/tests/queries/0_stateless/02207_ttl_move_if_exists.sql b/tests/queries/0_stateless/02207_ttl_move_if_exists.sql new file mode 100644 index 00000000000..ab17d343e49 --- /dev/null +++ b/tests/queries/0_stateless/02207_ttl_move_if_exists.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS t_ttl_move_if_exists; + +CREATE TABLE t_ttl_move_if_exists (d DateTime, a UInt32) +ENGINE = MergeTree ORDER BY tuple() +TTL d TO DISK IF EXISTS 'non_existing_disk'; + +SHOW CREATE TABLE t_ttl_move_if_exists; + +DROP TABLE IF EXISTS t_ttl_move_if_exists; diff --git a/tests/queries/0_stateless/02209_short_circuit_node_without_parents.reference b/tests/queries/0_stateless/02209_short_circuit_node_without_parents.reference new file mode 100644 index 00000000000..d00491fd7e5 --- /dev/null +++ b/tests/queries/0_stateless/02209_short_circuit_node_without_parents.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/02209_short_circuit_node_without_parents.sql b/tests/queries/0_stateless/02209_short_circuit_node_without_parents.sql new file mode 100644 index 00000000000..c20ca83591f --- /dev/null +++ b/tests/queries/0_stateless/02209_short_circuit_node_without_parents.sql @@ -0,0 +1,2 @@ +SELECT 1 FROM (SELECT arrayJoin(if(empty(range(number)), [1], [2])) from numbers(1)); + diff --git a/tests/queries/0_stateless/02210_append_to_dev_dull.reference b/tests/queries/0_stateless/02210_append_to_dev_dull.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02210_append_to_dev_dull.sql b/tests/queries/0_stateless/02210_append_to_dev_dull.sql new file mode 100644 index 00000000000..a8aaa2f05ab --- /dev/null +++ b/tests/queries/0_stateless/02210_append_to_dev_dull.sql @@ -0,0 +1,6 @@ +-- Tags: no-fasttest + +insert into table function file('/dev/null', 'Parquet', 'number UInt64') select * from numbers(10); +insert into table function file('/dev/null', 'ORC', 'number UInt64') select * from numbers(10); +insert into table function file('/dev/null', 'JSON', 'number UInt64') select * from numbers(10); + diff --git a/tests/queries/0_stateless/02210_toColumnTypeName_toLowCardinality_const.reference b/tests/queries/0_stateless/02210_toColumnTypeName_toLowCardinality_const.reference new file mode 100644 index 00000000000..2ac2f690f1b --- /dev/null +++ b/tests/queries/0_stateless/02210_toColumnTypeName_toLowCardinality_const.reference @@ -0,0 +1 @@ +Const(UInt8) diff --git a/tests/queries/0_stateless/02210_toColumnTypeName_toLowCardinality_const.sql b/tests/queries/0_stateless/02210_toColumnTypeName_toLowCardinality_const.sql new file mode 100644 index 00000000000..a71c3f30604 --- /dev/null +++ b/tests/queries/0_stateless/02210_toColumnTypeName_toLowCardinality_const.sql @@ -0,0 +1 @@ +SELECT toColumnTypeName(toLowCardinality(1)); diff --git a/tests/queries/0_stateless/02211_jsonl_format_extension.reference b/tests/queries/0_stateless/02211_jsonl_format_extension.reference new file mode 100644 index 00000000000..8b1acc12b63 --- /dev/null +++ b/tests/queries/0_stateless/02211_jsonl_format_extension.reference @@ -0,0 +1,10 @@ +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 diff --git a/tests/queries/0_stateless/02211_jsonl_format_extension.sql b/tests/queries/0_stateless/02211_jsonl_format_extension.sql new file mode 100644 index 00000000000..08fff5a11f5 --- /dev/null +++ b/tests/queries/0_stateless/02211_jsonl_format_extension.sql @@ -0,0 +1,3 @@ +-- Tags: no-fasttest +insert into table function file('data.jsonl', 'JSONEachRow', 'x UInt32') select * from numbers(10); +select * from file('data.jsonl'); diff --git a/tests/queries/0_stateless/02211_shcema_inference_from_stdin.reference b/tests/queries/0_stateless/02211_shcema_inference_from_stdin.reference new file mode 100644 index 00000000000..d176e0ee1ed --- /dev/null +++ b/tests/queries/0_stateless/02211_shcema_inference_from_stdin.reference @@ -0,0 +1,15 @@ +x Nullable(Float64) +0 +1 +2 +3 +4 +5 +6 +7 +8 +9 +c1 Nullable(String) +c2 Nullable(String) +c3 Nullable(String) +1 2 3 diff --git a/tests/queries/0_stateless/02211_shcema_inference_from_stdin.sh b/tests/queries/0_stateless/02211_shcema_inference_from_stdin.sh new file mode 100755 index 00000000000..2b469797f89 --- /dev/null +++ b/tests/queries/0_stateless/02211_shcema_inference_from_stdin.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +$CLICKHOUSE_LOCAL -q "select toUInt32(number) as x from numbers(10) format JSONEachRow" > data.jsoneachrow + +$CLICKHOUSE_LOCAL -q "desc table table" < data.jsoneachrow +$CLICKHOUSE_LOCAL -q "select * from table" < data.jsoneachrow + +rm data.jsoneachrow + +echo -e "1\t2\t3" | $CLICKHOUSE_LOCAL -q "desc table table" --file=- +echo -e "1\t2\t3" | $CLICKHOUSE_LOCAL -q "select * from table" --file=- + diff --git a/tests/queries/0_stateless/02212_cte_and_table_alias.reference b/tests/queries/0_stateless/02212_cte_and_table_alias.reference new file mode 100644 index 00000000000..1d3d2cc6415 --- /dev/null +++ b/tests/queries/0_stateless/02212_cte_and_table_alias.reference @@ -0,0 +1,4 @@ +5000 +5000 +5000 +5000 diff --git a/tests/queries/0_stateless/02212_cte_and_table_alias.sql b/tests/queries/0_stateless/02212_cte_and_table_alias.sql new file mode 100644 index 00000000000..ce0fba4bf56 --- /dev/null +++ b/tests/queries/0_stateless/02212_cte_and_table_alias.sql @@ -0,0 +1,41 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/19222 +SET enable_global_with_statement = 1; + +WITH t AS + ( + SELECT number AS n + FROM numbers(10000) + ) +SELECT count(*) +FROM t AS a +WHERE a.n < 5000; + +WITH t AS + ( + SELECT number AS n + FROM numbers(10000) + ) +SELECT count(*) +FROM t AS a +WHERE t.n < 5000; + + +SET enable_global_with_statement = 0; + +WITH t AS + ( + SELECT number AS n + FROM numbers(10000) + ) +SELECT count(*) +FROM t AS a +WHERE a.n < 5000; + +WITH t AS + ( + SELECT number AS n + FROM numbers(10000) + ) +SELECT count(*) +FROM t AS a +WHERE t.n < 5000; diff --git a/tests/queries/0_stateless/02220_array_join_format.reference b/tests/queries/0_stateless/02220_array_join_format.reference new file mode 100644 index 00000000000..b1978acfbfa --- /dev/null +++ b/tests/queries/0_stateless/02220_array_join_format.reference @@ -0,0 +1,11 @@ +SELECT + range_, + point_ +FROM +( + SELECT + range(0, 10) AS range_, + point_ + FROM system.one + ARRAY JOIN range(0, 10) AS point_ +) diff --git a/tests/queries/0_stateless/02220_array_join_format.sql b/tests/queries/0_stateless/02220_array_join_format.sql new file mode 100644 index 00000000000..afea6855877 --- /dev/null +++ b/tests/queries/0_stateless/02220_array_join_format.sql @@ -0,0 +1 @@ +explain syntax select * from (select range(0, 10) range_, point_ from system.one array join range_ as point_); diff --git a/tests/queries/0_stateless/02221_parallel_replicas_bug.reference b/tests/queries/0_stateless/02221_parallel_replicas_bug.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02221_parallel_replicas_bug.sh b/tests/queries/0_stateless/02221_parallel_replicas_bug.sh new file mode 100755 index 00000000000..b4ac6817a54 --- /dev/null +++ b/tests/queries/0_stateless/02221_parallel_replicas_bug.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + + +${CLICKHOUSE_CLIENT} --allow_experimental_parallel_reading_from_replicas=1 -nmT < "$CURDIR"/01099_parallel_distributed_insert_select.sql > /dev/null diff --git a/tests/queries/0_stateless/data_orc/corrupted.orc b/tests/queries/0_stateless/data_orc/corrupted.orc new file mode 100644 index 00000000000..08f7ab951f9 Binary files /dev/null and b/tests/queries/0_stateless/data_orc/corrupted.orc differ diff --git a/tests/queries/0_stateless/helpers/pure_http_client.py b/tests/queries/0_stateless/helpers/pure_http_client.py index 9f79c4ac529..3335f141bb5 100644 --- a/tests/queries/0_stateless/helpers/pure_http_client.py +++ b/tests/queries/0_stateless/helpers/pure_http_client.py @@ -14,22 +14,23 @@ class ClickHouseClient: def __init__(self, host = CLICKHOUSE_SERVER_URL_STR): self.host = host - def query(self, query, connection_timeout = 1500): + def query(self, query, connection_timeout=1500, settings=dict(), binary_result=False): NUMBER_OF_TRIES = 30 DELAY = 10 + params = { + 'timeout_before_checking_execution_speed': 120, + 'max_execution_time': 6000, + 'database': CLICKHOUSE_DATABASE, + } + + # Add extra settings to params + params = {**params, **settings} + for i in range(NUMBER_OF_TRIES): - r = requests.post( - self.host, - params = { - 'timeout_before_checking_execution_speed': 120, - 'max_execution_time': 6000, - 'database': CLICKHOUSE_DATABASE - }, - timeout = connection_timeout, - data = query) + r = requests.post(self.host, params=params, timeout=connection_timeout, data=query) if r.status_code == 200: - return r.text + return r.content if binary_result else r.text else: print('ATTENTION: try #%d failed' % i) if i != (NUMBER_OF_TRIES-1): @@ -44,9 +45,22 @@ class ClickHouseClient: df = pd.read_csv(io.StringIO(data), sep = '\t') return df - def query_with_data(self, query, content): - content = content.encode('utf-8') - r = requests.post(self.host, data=content) + def query_with_data(self, query, data, connection_timeout=1500, settings=dict()): + params = { + 'query': query, + 'timeout_before_checking_execution_speed': 120, + 'max_execution_time': 6000, + 'database': CLICKHOUSE_DATABASE, + } + + headers = { + "Content-Type": "application/binary" + } + + # Add extra settings to params + params = {**params, **settings} + + r = requests.post(self.host, params=params, timeout=connection_timeout, data=data, headers=headers) result = r.text if r.status_code == 200: return result diff --git a/uncrustify.cfg b/uncrustify.cfg deleted file mode 100644 index 9fefc1270de..00000000000 --- a/uncrustify.cfg +++ /dev/null @@ -1,252 +0,0 @@ -# Configuration file for Uncrustify code formatter. -# https://github.com/uncrustify/uncrustify -# -# Created with https://cdanu.github.io/uncrustify_config_preview/index.html -# -# You may apply it for your code with: -# uncrustify -l CPP -c uncrustify.cfg -f filename.cpp -# -# This config is in beta: it doesn't implement our style guide perfectly. -# It's not recommended to apply it for existing code base. - -newlines = lf -input_tab_size = 4 -output_tab_size = 4 -string_replace_tab_chars = true -utf8_bom = remove -utf8_byte = true -utf8_force = true -sp_arith = force -sp_assign = force -sp_cpp_lambda_assign = remove -sp_cpp_lambda_paren = remove -sp_assign_default = force -sp_enum_assign = force -sp_enum_colon = force -sp_pp_concat = force -sp_pp_stringify = remove -sp_bool = force -sp_compare = force -sp_inside_paren = remove -sp_paren_paren = remove -sp_paren_brace = force -sp_before_ptr_star = force -sp_between_ptr_star = remove -sp_after_ptr_star = force -sp_after_ptr_star_qualifier = force -sp_after_ptr_star_func = force -sp_ptr_star_paren = force -sp_before_ptr_star_func = force -sp_before_byref = force -sp_before_unnamed_byref = force -sp_after_byref = force -sp_after_byref_func = force -sp_before_byref_func = force -sp_template_angle = force -sp_before_angle = remove -sp_inside_angle = remove -sp_angle_colon = force -sp_after_angle = force -sp_angle_paren = remove -sp_angle_paren_empty = remove -sp_angle_word = force -sp_angle_shift = remove -sp_permit_cpp11_shift = true -sp_before_sparen = force -sp_inside_sparen = remove -sp_after_sparen = force -sp_sparen_brace = force -sp_special_semi = force -sp_before_semi_for = remove -sp_before_semi_for_empty = remove -sp_after_semi = force -sp_after_semi_for_empty = remove -sp_before_square = remove -sp_before_squares = remove -sp_inside_square = remove -sp_after_comma = force -sp_before_ellipsis = remove -sp_after_class_colon = force -sp_before_class_colon = force -sp_after_constr_colon = force -sp_before_constr_colon = force -sp_after_operator = remove -sp_after_operator_sym = remove -sp_after_cast = remove -sp_inside_paren_cast = remove -sp_cpp_cast_paren = remove -sp_sizeof_paren = remove -sp_inside_braces_enum = force -sp_inside_braces_struct = force -sp_inside_braces = force -sp_inside_braces_empty = remove -sp_type_func = force -sp_func_proto_paren = remove -sp_func_proto_paren_empty = remove -sp_func_def_paren = remove -sp_func_def_paren_empty = remove -sp_inside_fparens = remove -sp_inside_fparen = remove -sp_inside_tparen = remove -sp_after_tparen_close = remove -sp_square_fparen = remove -sp_fparen_brace = force -sp_func_call_paren = remove -sp_func_class_paren = remove -sp_func_class_paren_empty = remove -sp_return_paren = force -sp_attribute_paren = remove -sp_defined_paren = remove -sp_throw_paren = force -sp_after_throw = force -sp_catch_paren = force -sp_macro = add -sp_macro_func = add -sp_else_brace = force -sp_brace_else = force -sp_brace_typedef = force -sp_catch_brace = force -sp_brace_catch = force -sp_try_brace = force -sp_word_brace = remove -sp_word_brace_ns = force -sp_before_dc = remove -sp_after_dc = remove -sp_cond_colon = force -sp_cond_colon_before = force -sp_cond_colon_after = force -sp_cond_question = force -sp_cond_question_before = force -sp_cond_question_after = force -sp_cond_ternary_short = remove -sp_cmt_cpp_start = force -sp_cmt_cpp_doxygen = true -sp_cmt_cpp_qttr = true -sp_endif_cmt = force -sp_after_new = force -sp_between_new_paren = remove -sp_after_newop_paren = force -sp_inside_newop_paren = remove -sp_before_tr_emb_cmt = force -indent_columns = 4 -indent_with_tabs = 0 -indent_namespace = false -indent_namespace_limit = 100 -indent_class = true -indent_ctor_init_leading = 1 -indent_shift = true -indent_func_call_param = true -indent_func_def_param = true -indent_func_proto_param = true -indent_func_class_param = true -indent_func_ctor_var_param = true -indent_template_param = true -indent_member = 4 -indent_switch_case = 4 -indent_switch_pp = false -indent_label = 0 -indent_access_spec = -4 -indent_paren_close = 2 -indent_paren_after_func_def = true -indent_paren_after_func_decl = true -indent_paren_after_func_call = true -indent_align_assign = false -indent_token_after_brace = false -indent_cpp_lambda_body = true -indent_ternary_operator = 1 -nl_assign_leave_one_liners = true -nl_class_leave_one_liners = true -nl_enum_leave_one_liners = true -nl_getset_leave_one_liners = true -nl_func_leave_one_liners = true -nl_cpp_lambda_leave_one_liners = true -nl_cpp_ldef_brace = add -nl_if_leave_one_liners = true -nl_start_of_file = remove -nl_end_of_file = force -nl_enum_brace = add -nl_struct_brace = add -nl_union_brace = add -nl_if_brace = add -nl_brace_else = add -nl_else_brace = add -nl_else_if = remove -nl_before_if_closing_paren = remove -nl_try_brace = add -nl_for_brace = add -nl_catch_brace = add -nl_brace_catch = add -nl_while_brace = add -nl_do_brace = add -nl_brace_while = remove -nl_switch_brace = add -nl_multi_line_define = true -nl_before_case = true -nl_after_case = true -nl_case_colon_brace = add -nl_namespace_brace = add -nl_template_class = add -nl_class_brace = add -nl_enum_own_lines = add -nl_func_scope_name = remove -nl_func_paren = remove -nl_func_def_paren = remove -nl_func_call_paren = remove -nl_func_call_paren_empty = remove -nl_func_decl_start_multi_line = true -nl_func_def_start_multi_line = true -nl_func_decl_args_multi_line = true -nl_func_def_args_multi_line = true -nl_func_decl_end = remove -nl_func_def_end = remove -nl_func_decl_empty = remove -nl_func_def_empty = remove -nl_func_call_empty = remove -nl_func_call_start_multi_line = true -nl_func_call_args_multi_line = true -nl_fdef_brace = add -nl_after_semicolon = true -nl_constr_colon = force -nl_split_if_one_liner = true -nl_split_for_one_liner = true -nl_split_while_one_liner = true -nl_max = 3 -nl_max_blank_in_func = 2 -nl_after_func_proto = 1 -nl_after_func_proto_group = 2 -nl_after_func_class_proto = 1 -nl_after_func_class_proto_group = 2 -nl_before_func_body_def = 1 -nl_before_func_body_proto = 1 -nl_after_func_body = 3 -nl_after_func_body_class = 3 -nl_after_func_body_one_liner = 1 -nl_after_multiline_comment = true -nl_after_struct = 3 -nl_before_class = 3 -nl_after_class = 3 -nl_before_access_spec = 2 -nl_after_access_spec = 1 -nl_after_try_catch_finally = 1 -eat_blanks_after_open_brace = true -eat_blanks_before_close_brace = true -nl_remove_extra_newlines = 1 -nl_after_return = true -pos_constr_comma = lead_break -pos_constr_colon = lead_force -code_width = 160 -ls_func_split_full = true -ls_code_width = true -align_left_shift = false -cmt_convert_tab_to_spaces = true -mod_full_brace_for = remove -mod_full_brace_if = remove -mod_full_brace_if_chain = true -mod_full_brace_while = remove -mod_paren_on_return = remove -mod_remove_extra_semicolon = true -mod_remove_empty_return = true -align_func_params = true -align_func_params_thresh = 140 -sp_inside_type_brace_init_lst = remove -nl_constr_init_args = add diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt index 7822f47ff88..51300472ed1 100644 --- a/utils/CMakeLists.txt +++ b/utils/CMakeLists.txt @@ -26,7 +26,6 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS) add_subdirectory (zookeeper-adjust-block-numbers-to-parts) add_subdirectory (wikistat-loader) add_subdirectory (check-marks) - add_subdirectory (convert-month-partitioned-parts) add_subdirectory (checksum-for-compressed-block) add_subdirectory (db-generator) add_subdirectory (wal-dump) diff --git a/utils/c++expr b/utils/c++expr index ec24b97ebc9..c498e780d05 100755 --- a/utils/c++expr +++ b/utils/c++expr @@ -3,7 +3,7 @@ set -e usage() { cat <&2 -USAGE: c++expr [-c CXX | -C | -I] [-i INCLUDE] [-b STEPS] [-t TESTS] [-o FILE] [-O CXX_OPTS...] [-g 'GLOBAL CODE'] 'MAIN CODE' +USAGE: c++expr [-c CXX | -C | -I] [-i INCLUDE] [-l LIB] [-b STEPS] [-t TESTS] [-o FILE] [-O CXX_OPTS...] [-g 'GLOBAL CODE'] 'MAIN CODE' OPTIONS: -c CXX use specified c++ compiler -C use cmake @@ -15,6 +15,19 @@ OPTIONS: -t TESTS_NUM make program to benchmark specified code snippet and run TESTS_NUM tests -o FILE do not run, just save binary executable file -O CXX_OPTS forward option compiler (e.g. -O "-O3 -std=c++20") +EXAMPLES: + $ c++expr -g 'int fib(int n) { return n < 2 ? n : fib(n-2) + fib(n-1); }' 'OUT(fib(10)) OUT(fib(20)) OUT(fib(30))' + fib(10) -> 55 + fib(20) -> 6765 + fib(30) -> 832040 + $ c++expr -I -i Interpreters/Context.h 'OUT(sizeof(DB::Context))' + sizeof(DB::Context) -> 7776 + $ c++expr -I -i Common/Stopwatch.h -b 10000 'Stopwatch sw;' + Steps per test: 10000 + Test #0: 0.0178 us 5.61798e+07 sps + ... + Test #4: 0.0179 us 5.58659e+07 sps + Average: 0.0179 us 5.58659e+07 sps EOF exit 1 } @@ -37,7 +50,7 @@ CMD_PARAMS= # Parse command line # -if [ "$1" == "--help" ]; then usage; fi +if [ "$1" == "--help" ] || [ -z "$1" ]; then usage; fi while getopts "vc:CIi:l:b:t:o:O:g:" OPT; do case "$OPT" in v) set -x; ;; diff --git a/utils/check-style/check-workflows b/utils/check-style/check-workflows old mode 100644 new mode 100755 index 1a0c0f9ecbe..c0399829c28 --- a/utils/check-style/check-workflows +++ b/utils/check-style/check-workflows @@ -1,5 +1,6 @@ #!/usr/bin/env bash -act --list 1>/dev/null 2>&1 || act --list 2>&1 +GIT_ROOT=$(git rev-parse --show-cdup) +act --list --directory="$GIT_ROOT" 1>/dev/null 2>&1 || act --list --directory="$GIT_ROOT" 2>&1 actionlint diff --git a/utils/clickhouse-diagnostics/README.md b/utils/clickhouse-diagnostics/README.md index 991efefdf5a..a6f8ed298dd 100644 --- a/utils/clickhouse-diagnostics/README.md +++ b/utils/clickhouse-diagnostics/README.md @@ -1,3 +1,18 @@ +## Download + +Cloning whole repo will take a lot of time and disk space. The following commands will download only this directory. + +* Requires Git 2.19 + +``` +# mkdir chdiag +# cd chdiag +# git clone --depth 1 --filter=blob:none --no-checkout https://github.com/ClickHouse/ClickHouse +# cd ClickHouse +# git sparse-checkout set utils/clickhouse-diagnostics +# git checkout master -- utils/clickhouse-diagnostics +``` + ## Installation ``` diff --git a/utils/clickhouse-diagnostics/clickhouse-diagnostics b/utils/clickhouse-diagnostics/clickhouse-diagnostics index ffddee0bdc4..83c0af9cd11 100644 --- a/utils/clickhouse-diagnostics/clickhouse-diagnostics +++ b/utils/clickhouse-diagnostics/clickhouse-diagnostics @@ -953,7 +953,7 @@ def parse_version(version): """ Parse version string. """ - return [int(x) for x in version.strip().split('.')] + return [int(x) for x in version.strip().split('.') if x.isnumeric()] if __name__ == '__main__': diff --git a/utils/convert-month-partitioned-parts/CMakeLists.txt b/utils/convert-month-partitioned-parts/CMakeLists.txt deleted file mode 100644 index ea6429a0610..00000000000 --- a/utils/convert-month-partitioned-parts/CMakeLists.txt +++ /dev/null @@ -1,2 +0,0 @@ -add_executable (convert-month-partitioned-parts main.cpp) -target_link_libraries(convert-month-partitioned-parts PRIVATE clickhouse_aggregate_functions dbms clickhouse_parsers boost::program_options) diff --git a/utils/convert-month-partitioned-parts/main.cpp b/utils/convert-month-partitioned-parts/main.cpp deleted file mode 100644 index a6829d79726..00000000000 --- a/utils/convert-month-partitioned-parts/main.cpp +++ /dev/null @@ -1,148 +0,0 @@ -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -#include -#include - -namespace DB -{ - -namespace ErrorCodes -{ - extern const int DIRECTORY_ALREADY_EXISTS; - extern const int BAD_DATA_PART_NAME; - extern const int NO_FILE_IN_DATA_PART; -} - -void run(String part_path, String date_column, String dest_path) -{ - std::shared_ptr disk = std::make_shared("local", "/", 0); - auto old_part_path = Poco::Path::forDirectory(part_path); - const String & old_part_name = old_part_path.directory(old_part_path.depth() - 1); - String old_part_path_str = old_part_path.toString(); - - auto part_info = MergeTreePartInfo::fromPartName(old_part_name, MergeTreeDataFormatVersion(0)); - String new_part_name = part_info.getPartName(); - - auto new_part_path = Poco::Path::forDirectory(dest_path); - new_part_path.pushDirectory(new_part_name); - if (Poco::File(new_part_path).exists()) - throw Exception("Destination part directory `" + new_part_path.toString() + "` already exists", - ErrorCodes::DIRECTORY_ALREADY_EXISTS); - - DayNum min_date; - DayNum max_date; - MergeTreePartInfo::parseMinMaxDatesFromPartName(old_part_name, min_date, max_date); - - const auto & time_zone = DateLUT::instance(); - UInt32 yyyymm = time_zone.toNumYYYYMM(min_date); - if (yyyymm != time_zone.toNumYYYYMM(max_date)) - throw Exception("Part " + old_part_name + " spans different months", - ErrorCodes::BAD_DATA_PART_NAME); - - ReadBufferFromFile checksums_in(old_part_path_str + "checksums.txt", 4096); - MergeTreeDataPartChecksums checksums; - checksums.read(checksums_in); - - auto date_col_checksum_it = checksums.files.find(date_column + ".bin"); - if (date_col_checksum_it == checksums.files.end()) - throw Exception("Couldn't find checksum for the date column .bin file `" + date_column + ".bin`", - ErrorCodes::NO_FILE_IN_DATA_PART); - - UInt64 rows = date_col_checksum_it->second.uncompressed_size / DataTypeDate().getSizeOfValueInMemory(); - - auto new_tmp_part_path = Poco::Path::forDirectory(dest_path); - new_tmp_part_path.pushDirectory("tmp_convert_" + new_part_name); - String new_tmp_part_path_str = new_tmp_part_path.toString(); - try - { - Poco::File(new_tmp_part_path).remove(/* recursive = */ true); - } - catch (const Poco::FileNotFoundException &) - { - /// If the file is already deleted, do nothing. - } - localBackup(disk, old_part_path.toString(), new_tmp_part_path.toString(), {}); - - WriteBufferFromFile count_out(new_tmp_part_path_str + "count.txt", 4096); - HashingWriteBuffer count_out_hashing(count_out); - writeIntText(rows, count_out_hashing); - count_out_hashing.next(); - checksums.files["count.txt"].file_size = count_out_hashing.count(); - checksums.files["count.txt"].file_hash = count_out_hashing.getHash(); - - IMergeTreeDataPart::MinMaxIndex minmax_idx(min_date, max_date); - Names minmax_idx_columns = {date_column}; - DataTypes minmax_idx_column_types = {std::make_shared()}; - minmax_idx.store(minmax_idx_columns, minmax_idx_column_types, disk, new_tmp_part_path_str, checksums); - - Block partition_key_sample{{nullptr, std::make_shared(), makeASTFunction("toYYYYMM", std::make_shared(date_column))->getColumnName()}}; - - MergeTreePartition partition(yyyymm); - partition.store(partition_key_sample, disk, new_tmp_part_path_str, checksums); - String partition_id = partition.getID(partition_key_sample); - - Poco::File(new_tmp_part_path_str + "checksums.txt").setWriteable(); - WriteBufferFromFile checksums_out(new_tmp_part_path_str + "checksums.txt", 4096); - checksums.write(checksums_out); - checksums_in.close(); - checksums_out.close(); - - Poco::File(new_tmp_part_path).renameTo(new_part_path.toString()); -} - -} - -int main(int argc, char ** argv) -try -{ - boost::program_options::options_description desc("Allowed options"); - desc.add_options() - ("help,h", "produce help message") - ("part", boost::program_options::value()->required(), - "part directory to convert") - ("date-column", boost::program_options::value()->required(), - "name of the date column") - ("to", boost::program_options::value()->required(), - "destination directory") - ; - - boost::program_options::variables_map options; - boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options); - - if (options.count("help") || options.size() < 3) - { - std::cout - << "Convert a MergeTree part from the old-style month-partitioned table " - << "(e.g. 20140317_20140323_2_2_0) to the format suitable for ATTACH'ing to a custom-partitioned " - << "table (201403_2_2_0)." << std::endl << std::endl; - std::cout << desc << std::endl; - return 1; - } - - auto part_path = options.at("part").as(); - auto date_column = options.at("date-column").as(); - auto dest_path = options.at("to").as(); - - DB::run(part_path, date_column, dest_path); - - return 0; -} -catch (...) -{ - std::cerr << DB::getCurrentExceptionMessage(true) << '\n'; - throw; -} diff --git a/utils/list-versions/list-versions.sh b/utils/list-versions/list-versions.sh index 01364c78f83..8d4f286124e 100755 --- a/utils/list-versions/list-versions.sh +++ b/utils/list-versions/list-versions.sh @@ -1,3 +1,5 @@ #!/bin/bash -git tag --list | grep -P 'v.+-(stable|lts)' | sort -V | xargs git show --format='%ai' | awk '/^v/ { version = $1 } /^[0-9]+/ { if (version) { date = $1 } } { if (version && date) { print version "\t" date; version = ""; date = ""; } }' | tac +# refname:strip=2: default tag name when format is not set +# creatordate is always defined for all tags +git tag --list 'v*-lts' 'v*-stable' --format='%(refname:short) %(creatordate:short)' | sort -rV diff --git a/utils/list-versions/version_date.tsv b/utils/list-versions/version_date.tsv index 539aa6f1b19..cff412d9fd8 100644 --- a/utils/list-versions/version_date.tsv +++ b/utils/list-versions/version_date.tsv @@ -1,56 +1,87 @@ -v21.9.4.35-stable 2021-09-22 -v21.9.3.30-stable 2021-09-16 +v22.1.3.7-stable 2022-01-23 +v22.1.2.2-stable 2022-01-19 +v21.12.4.1-stable 2022-01-23 +v21.12.3.32-stable 2021-12-27 +v21.12.2.17-stable 2021-12-16 +v21.11.11.1-stable 2022-01-23 +v21.11.10.1-stable 2022-01-12 +v21.11.9.1-stable 2021-12-27 +v21.11.8.4-stable 2021-12-22 +v21.11.7.9-stable 2021-12-15 +v21.11.6.7-stable 2021-12-10 +v21.11.5.33-stable 2021-12-02 +v21.11.4.14-stable 2021-11-17 +v21.11.3.6-stable 2021-11-11 +v21.11.2.2-stable 2021-11-09 +v21.10.6.2-stable 2022-01-24 +v21.10.5.3-stable 2021-12-10 +v21.10.4.26-stable 2021-12-02 +v21.10.3.9-stable 2021-11-17 +v21.10.2.15-stable 2021-10-18 +v21.9.6.24-stable 2021-12-02 +v21.9.5.16-stable 2021-10-19 +v21.9.4.35-stable 2021-09-24 +v21.9.3.30-stable 2021-09-17 v21.9.2.17-stable 2021-09-09 -v21.8.8.29-lts 2021-09-28 -v21.8.7.22-lts 2021-09-22 -v21.8.6.15-lts 2021-09-16 +v21.8.14.5-lts 2022-01-26 +v21.8.13.6-lts 2021-12-27 +v21.8.12.29-lts 2021-12-02 +v21.8.11.4-lts 2021-11-17 +v21.8.10.19-lts 2021-10-21 +v21.8.9.13-lts 2021-10-19 +v21.8.8.29-lts 2021-09-29 +v21.8.7.22-lts 2021-09-24 +v21.8.6.15-lts 2021-09-17 v21.8.5.7-lts 2021-09-02 -v21.8.4.51-lts 2021-08-17 +v21.8.4.51-lts 2021-08-18 v21.8.3.44-lts 2021-08-12 -v21.7.11.3-stable 2021-09-23 -v21.7.10.4-stable 2021-09-16 +v21.7.11.3-stable 2021-09-24 +v21.7.10.4-stable 2021-09-18 v21.7.9.7-stable 2021-09-02 -v21.7.8.58-stable 2021-08-17 -v21.7.7.47-stable 2021-08-09 +v21.7.8.58-stable 2021-08-18 +v21.7.7.47-stable 2021-08-10 v21.7.6.39-stable 2021-08-06 -v21.7.5.29-stable 2021-07-28 -v21.7.4.18-stable 2021-07-17 -v21.7.3.14-stable 2021-07-13 +v21.7.5.29-stable 2021-07-29 +v21.7.4.18-stable 2021-07-19 +v21.7.3.14-stable 2021-07-14 v21.7.2.7-stable 2021-07-09 -v21.6.9.7-stable 2021-09-02 -v21.6.8.62-stable 2021-07-13 -v21.6.7.57-stable 2021-07-09 -v21.6.6.51-stable 2021-07-02 +v21.6.9.7-stable 2021-09-03 +v21.6.8.62-stable 2021-07-16 +v21.6.7.57-stable 2021-07-10 +v21.6.6.51-stable 2021-07-04 v21.6.5.37-stable 2021-06-19 v21.6.4.26-stable 2021-06-11 -v21.6.3.14-stable 2021-06-04 +v21.6.3.14-stable 2021-06-05 v21.5.9.4-stable 2021-07-10 -v21.5.8.21-stable 2021-07-02 -v21.5.7.9-stable 2021-06-22 -v21.5.6.6-stable 2021-05-29 +v21.5.8.21-stable 2021-07-04 +v21.5.7.9-stable 2021-06-23 +v21.5.6.6-stable 2021-05-30 v21.5.5.12-stable 2021-05-20 -v21.4.7.3-stable 2021-05-19 -v21.4.6.55-stable 2021-04-30 +v21.4.7.3-stable 2021-05-20 +v21.4.6.55-stable 2021-05-01 v21.4.5.46-stable 2021-04-24 v21.4.4.30-stable 2021-04-16 v21.4.3.21-stable 2021-04-12 -v21.3.17.2-lts 2021-09-16 -v21.3.16.5-lts 2021-09-03 +v21.3.20.1-lts 2022-01-26 +v21.3.19.1-lts 2021-12-10 +v21.3.18.4-lts 2021-10-21 +v21.3.17.2-lts 2021-09-17 +v21.3.16.5-lts 2021-09-04 v21.3.15.4-stable 2021-07-10 -v21.3.14.1-lts 2021-07-01 -v21.3.13.9-lts 2021-06-22 -v21.3.12.2-lts 2021-05-25 -v21.3.11.5-lts 2021-05-14 -v21.3.10.1-lts 2021-05-09 -v21.3.9.83-lts 2021-04-28 +v21.3.14.1-lts 2021-07-04 +v21.3.13.9-lts 2021-06-23 +v21.3.12.2-lts 2021-05-26 +v21.3.11.5-lts 2021-05-16 +v21.3.10.1-lts 2021-05-10 +v21.3.9.83-lts 2021-05-01 v21.3.8.76-lts 2021-04-24 -v21.3.7.62-stable 2021-04-16 +v21.3.7.62-stable 2021-04-17 v21.3.6.55-lts 2021-04-12 v21.3.5.42-lts 2021-04-07 v21.3.4.25-lts 2021-03-28 v21.3.3.14-lts 2021-03-19 v21.3.2.5-lts 2021-03-12 -v21.2.10.48-stable 2021-04-16 +v21.2.10.48-stable 2021-04-17 v21.2.9.41-stable 2021-04-12 v21.2.8.31-stable 2021-04-07 v21.2.7.11-stable 2021-03-28 @@ -93,8 +124,8 @@ v20.9.5.5-stable 2020-11-13 v20.9.4.76-stable 2020-10-29 v20.9.3.45-stable 2020-10-09 v20.9.2.20-stable 2020-09-22 -v20.8.19.4-stable 2021-07-10 -v20.8.18.32-lts 2021-04-16 +v20.8.19.4-stable 2021-07-11 +v20.8.18.32-lts 2021-04-17 v20.8.17.25-lts 2021-04-08 v20.8.16.20-lts 2021-04-06 v20.8.15.11-lts 2021-04-01 @@ -239,97 +270,104 @@ v19.9.2.4-stable 2019-06-24 v19.8.3.8-stable 2019-06-11 v19.7.5.29-stable 2019-07-05 v19.7.5.27-stable 2019-06-09 -v19.7.3.9-stable 2019-05-05 +v19.7.3.9-stable 2019-05-27 v19.6.3.18-stable 2019-06-15 -v19.6.2.11-stable 2019-04-30 -v19.5.4.22-stable 2019-04-30 -v19.5.3.8-stable 2019-04-17 +v19.6.2.11-stable 2019-05-14 +v19.5.4.22-stable 2019-05-13 +v19.5.3.8-stable 2019-04-18 v19.5.2.6-stable 2019-04-15 -v19.4.5.35-stable 2019-05-05 -v19.4.4.33-stable 2019-04-16 -v19.4.3.11-stable 2019-04-01 -v19.4.2.7-stable 2019-03-29 +v19.4.5.35-stable 2019-05-13 +v19.4.4.33-stable 2019-04-17 +v19.4.3.11-stable 2019-04-02 +v19.4.2.7-stable 2019-03-30 v19.4.1.3-stable 2019-03-19 v19.4.0.49-stable 2019-03-09 -v19.3.9.12-stable 2019-04-01 -v19.3.8.6-stable 2019-03-04 -v19.3.7-stable 2019-03-11 +v19.3.9.12-stable 2019-04-02 +v19.3.8.6-stable 2019-03-19 +v19.3.7-stable 2019-03-12 v19.3.6-stable 2019-03-02 -v19.3.5-stable 2019-02-20 -v19.3.4-stable 2019-02-15 +v19.3.5-stable 2019-02-21 +v19.3.4-stable 2019-02-16 v19.3.3-stable 2019-02-13 -v19.1.16.79-stable 2019-03-22 -v19.1.15.73-stable 2019-03-04 +v19.1.16.79-stable 2019-04-02 +v19.1.15.73-stable 2019-03-19 v19.1.14-stable 2019-03-14 v19.1.13-stable 2019-03-12 v19.1.10-stable 2019-03-03 -v19.1.9-stable 2019-02-20 -v19.1.8-stable 2019-02-15 -v19.1.7-stable 2019-02-13 +v19.1.9-stable 2019-02-21 +v19.1.8-stable 2019-02-16 +v19.1.7-stable 2019-02-15 v19.1.6-stable 2019-01-24 -v19.1.5-stable 2019-01-22 -v18.16.1-stable 2018-12-20 -v18.16.0-stable 2018-12-14 +v19.1.5-stable 2019-01-23 +v18.16.1-stable 2018-12-21 +v18.16.0-stable 2018-12-15 v18.14.19-stable 2018-12-19 v18.14.18-stable 2018-12-04 -v18.14.17-stable 2018-11-29 +v18.14.17-stable 2018-11-30 v18.14.15-stable 2018-11-21 -v18.14.14-stable 2018-11-20 -v18.14.13-stable 2018-11-07 -v18.14.12-stable 2018-11-01 -v18.14.11-stable 2018-10-26 -v18.14.10-stable 2018-10-23 +v18.14.14-stable 2018-11-21 +v18.14.13-stable 2018-11-08 +v18.14.12-stable 2018-11-02 +v18.14.11-stable 2018-10-29 +v18.14.10-stable 2018-10-24 v18.14.9-stable 2018-10-16 v18.14.8-stable 2018-10-13 v18.12.17-stable 2018-09-16 v18.12.14-stable 2018-09-13 -v18.12.13-stable 2018-09-10 -v1.1.54390-stable 2018-07-06 -v1.1.54388-stable 2018-06-27 +v18.12.13-stable 2018-09-11 +v18.10.3-stable 2018-08-13 +v18.6.0-stable 2018-08-01 +v18.5.1-stable 2018-07-31 +v18.4.0-stable 2018-07-28 +v18.1.0-stable 2018-07-20 +v1.1.54394-stable 2018-07-12 +v1.1.54390-stable 2018-07-09 +v1.1.54388-stable 2018-06-28 v1.1.54385-stable 2018-06-01 -v1.1.54383-stable 2018-05-18 -v1.1.54381-stable 2018-04-26 -v1.1.54380-stable 2018-04-20 -v1.1.54378-stable 2018-04-13 +v1.1.54383-stable 2018-05-22 +v1.1.54381-stable 2018-05-14 +v1.1.54380-stable 2018-04-21 +v1.1.54378-stable 2018-04-16 v1.1.54370-stable 2018-03-16 -v1.1.54362-stable 2018-03-10 +v1.1.54362-stable 2018-03-11 v1.1.54358-stable 2018-03-08 -v1.1.54343-stable 2018-01-24 +v1.1.54343-stable 2018-02-08 v1.1.54342-stable 2018-01-22 -v1.1.54337-stable 2018-01-17 -v1.1.54336-stable 2018-01-16 -v1.1.54335-stable 2018-01-16 -v1.1.54327-stable 2017-12-20 -v1.1.54318-stable 2017-11-30 +v1.1.54337-stable 2018-01-18 +v1.1.54336-stable 2018-01-17 +v1.1.54335-stable 2018-01-17 +v1.1.54327-stable 2017-12-21 +v1.1.54318-stable 2017-12-01 v1.1.54310-stable 2017-11-01 -v1.1.54304-stable 2017-10-19 -v1.1.54292-stable 2017-09-20 -v1.1.54289-stable 2017-09-13 -v1.1.54284-stable 2017-08-29 +v1.1.54304-stable 2017-10-24 +v1.1.54292-stable 2017-09-27 +v1.1.54289-stable 2017-09-14 +v1.1.54284-stable 2017-08-30 v1.1.54282-stable 2017-08-23 v1.1.54276-stable 2017-08-16 -v1.1.54245-stable 2017-07-04 -v1.1.54242-stable 2017-06-26 -v1.1.54236-stable 2017-05-23 -v1.1.54231-stable 2017-04-29 -v1.1.54198-stable 2017-03-28 -v1.1.54190-stable 2017-03-21 +v1.1.54245-stable 2017-07-05 +v1.1.54242-stable 2017-06-27 +v1.1.54236-stable 2017-05-25 +v1.1.54231-stable 2017-05-15 +v1.1.54198-stable 2017-03-29 +v1.1.54190-stable 2017-03-22 v1.1.54188-stable 2017-03-17 -v1.1.54181-stable 2017-03-09 -v1.1.54165-stable 2017-02-14 -v1.1.54159-stable 2017-02-03 -v1.1.54144-stable 2017-01-31 +v1.1.54181-stable 2017-03-11 +v1.1.54165-stable 2017-02-15 +v1.1.54159-stable 2017-02-07 +v1.1.54144-stable 2017-02-01 v1.1.54135-stable 2017-01-20 -v1.1.54134-stable 2017-01-14 -v1.1.54133-stable 2017-01-12 -v1.1.54127-stable 2016-12-22 -v1.1.54112-stable 2016-12-08 -v1.1.54083-stable 2016-11-28 +v1.1.54134-stable 2017-01-16 +v1.1.54133-stable 2017-01-13 +v1.1.54127-stable 2016-12-23 +v1.1.54112-stable 2016-12-13 +v1.1.54083-stable 2016-12-01 v1.1.54080-stable 2016-11-25 -v1.1.54074-stable 2016-11-23 -v1.1.54046-stable 2016-10-27 -v1.1.54030-stable 2016-10-24 -v1.1.54023-stable 2016-10-07 -v1.1.54022-stable 2016-09-26 -v1.1.54020-stable 2016-09-19 -v1.1.54019-stable 2016-09-07 +v1.1.54074-stable 2016-11-24 +v1.1.54046-stable 2016-11-08 +v1.1.54030-stable 2016-10-25 +v1.1.54023-stable 2016-10-10 +v1.1.54022-stable 2016-09-28 +v1.1.54020-stable 2016-09-21 +v1.1.54019-stable 2016-09-14 +v1.1.54011-stable 2016-08-18 diff --git a/website/css/base.css b/website/css/base.css index b65c8e869f4..d81b231789a 100644 --- a/website/css/base.css +++ b/website/css/base.css @@ -193,3 +193,11 @@ a.btn-outline-yellow { .height-20 { height: 20px; } + +ol.default{ + margin-top: 10px; +} + +ol.default li{ + margin-bottom: 10px; +} diff --git a/website/images/photos/artur-filatenkov.jpg b/website/images/photos/artur-filatenkov.jpg new file mode 100644 index 00000000000..4fdc7758268 Binary files /dev/null and b/website/images/photos/artur-filatenkov.jpg differ diff --git a/website/images/photos/bastian-spanneberg.jpg b/website/images/photos/bastian-spanneberg.jpg new file mode 100644 index 00000000000..95c9e5b73ac Binary files /dev/null and b/website/images/photos/bastian-spanneberg.jpg differ diff --git a/website/images/photos/manas-alekar.jpg b/website/images/photos/manas-alekar.jpg new file mode 100644 index 00000000000..307a860455f Binary files /dev/null and b/website/images/photos/manas-alekar.jpg differ diff --git a/website/images/photos/marcelo-rodriguez.jpg b/website/images/photos/marcelo-rodriguez.jpg new file mode 100644 index 00000000000..a25be5309da Binary files /dev/null and b/website/images/photos/marcelo-rodriguez.jpg differ diff --git a/website/images/photos/martin-choluj.jpg b/website/images/photos/martin-choluj.jpg new file mode 100644 index 00000000000..200685ab2fe Binary files /dev/null and b/website/images/photos/martin-choluj.jpg differ diff --git a/website/images/photos/mikhail-fursov.jpg b/website/images/photos/mikhail-fursov.jpg new file mode 100644 index 00000000000..82969fed20e Binary files /dev/null and b/website/images/photos/mikhail-fursov.jpg differ diff --git a/website/support/agreement/index.html b/website/support/agreement/index.html new file mode 100644 index 00000000000..59e5ca9ab75 --- /dev/null +++ b/website/support/agreement/index.html @@ -0,0 +1,27 @@ +{% set prefetch_items = [ + ('/docs/en/', 'document') +] %} + +{% extends "templates/base.html" %} + +{% block extra_meta %} + +{% include "templates/common_fonts.html" %} +{% endblock %} + +{% block nav %} + +{% include "templates/global/nav.html" %} + +{% endblock %} + +{% block content %} + +{% include "templates/support/agreement-hero.html" %} + +{% include "templates/support/agreement-content.html" %} + +{% include "templates/global/newsletter.html" %} +{% include "templates/global/github_stars.html" %} + +{% endblock %} diff --git a/website/support/policy/index.html b/website/support/policy/index.html new file mode 100644 index 00000000000..babff39a747 --- /dev/null +++ b/website/support/policy/index.html @@ -0,0 +1,27 @@ +{% set prefetch_items = [ + ('/docs/en/', 'document') +] %} + +{% extends "templates/base.html" %} + +{% block extra_meta %} + +{% include "templates/common_fonts.html" %} +{% endblock %} + +{% block nav %} + +{% include "templates/global/nav.html" %} + +{% endblock %} + +{% block content %} + +{% include "templates/support/policy-hero.html" %} + +{% include "templates/support/policy-content.html" %} + +{% include "templates/global/newsletter.html" %} +{% include "templates/global/github_stars.html" %} + +{% endblock %} diff --git a/website/templates/company/team.html b/website/templates/company/team.html index c4e18e36c96..c2e6bfe496d 100644 --- a/website/templates/company/team.html +++ b/website/templates/company/team.html @@ -9,6 +9,19 @@
+ + + +

+ {{ _('Manas Alekar') }} +

+

+ {{ _('Senior Cloud SWE') }} +

+ +
+
+ @@ -43,7 +56,7 @@ {{ _('Ivan Blinkov') }}

- {{ _('VP, Product') }} + {{ _('VP, Technical Program Management') }}

@@ -99,6 +112,19 @@ {{ _('Software Engineer') }}

+
+
+ + + + +

+ {{ _('Martin Choluj') }} +

+

+ {{ _('Vice President, Security') }} +

+
@@ -125,6 +151,19 @@ {{ _('Core SWE') }}

+
+
+ + + + +

+ {{ _('Artur Filatenkov') }} +

+

+ {{ _('Associate Core Software Engineer') }} +

+
@@ -138,6 +177,19 @@ {{ _('Senior Director, Business Technology') }}

+
+
+ +
+ +
+

+ {{ _('Mikhail Fursov') }} +

+

+ {{ _('Principal UX/UI Engineer') }} +

+
@@ -437,6 +489,19 @@ {{ _('Director, Global Learning') }}

+
+
+ + + + +

+ {{ _('Marcelo Rodriguez') }} +

+

+ {{ _('Sr Support Engineer') }} +

+
@@ -476,6 +541,19 @@ {{ _('Site Reliability Engineer') }}

+
+
+ + + + +

+ {{ _('Bastian Spanneberg') }} +

+

+ {{ _('Site Reliability Engineer') }} +

+
diff --git a/website/templates/support/agreement-content.html b/website/templates/support/agreement-content.html new file mode 100644 index 00000000000..4ca64e69599 --- /dev/null +++ b/website/templates/support/agreement-content.html @@ -0,0 +1,120 @@ +
+
+ +

This ClickHouse Subscription Agreement, including all referenced URLs, which are incorporated herein by reference (collectively, this “Agreement”), is entered into as of the date on which an applicable Order Form is fully executed (“Effective Date”), by and between the ClickHouse entity ("ClickHouse") set forth on such Order Form, and the entity identified thereon as the “Customer” (“Customer”).

+ +

1. DEFINITIONS
Capitalized terms used herein have the meaning ascribed below, or where such terms are first used, as applicable.

+ +

1.1 "Affiliate" means, with respect to a party, any entity that controls, is controlled by, or which is under common control with, such party, where "control" means ownership of at least fifty percent (50%) of the outstanding voting shares of the entity, or the contractual right to establish policy for, and manage the operations of, the entity.

+ +

1.2 "Order Form" means an ordering document provided by ClickHouse pursuant to which Customer purchases Subscriptions under this Agreement.

+ +

1.3 "Qualifying PO" means a purchase order issued by customer for the purpose of purchasing a Subscription, which (i) references the number of an applicable Order Form provided to Customer by ClickHouse and (ii) clearly states the purchase order is subject to the terms and conditions of this Agreement.

+ +

1.4 "Software" means the ClickHouse software of the same name that is licensed for use under the Apache 2.0 license.

+ +

1.5 "Subscription" means Customer's right, for a fixed period of time, to receive Support Services, as set forth in the applicable Order Form.

+ +

1.6 "Subscription Term" means the period of time for which a Subscription is valid, as further described in Section 7.1 of this Agreement.

+ +

1.7 "Support Services" means maintenance and support services for the Software, as more fully described in the Support Services Policy.

+ +

1.8 "Support Services Policy" means ClickHouse's support services policy as further described at https://clickhouse.com/support/policy/. ClickHouse reserves the right to reasonably modify the Support Services Policy during a Subscription Term, provided however, ClickHouse shall not materially diminish the level of Support Services during a Subscription Term. The effective date of each version of the Support Services Policy will be stated thereon, and ClickHouse agrees to archive copies of each version, and make the same available to Customer upon written request (e-mail sufficient). The parties agree that the Support Services Policy is hereby incorporated into these terms and conditions by this reference.

+ +

2. AGREEMENT SCOPE AND PERFORMANCE OF SUPPORT SERVICES

+ +

2.1 Agreement Scope. This Agreement includes terms and conditions applicable to Subscriptions for Support Services purchased under each Order Form entered into by the parties under Section 2.2 below, which Support Services may be used by Customer solely for Internal use and in connection with the use case(s) set forth on the applicable Order Form.

+ +

2.2 Order for Support Services Subscriptions. Orders for Subscriptions may be placed by Customer through (1) the execution of Order Forms with ClickHouse or (2) issuance by Customer of a Qualifying PO, which will be deemed to constitute, for the purposes of this Agreement, the execution by Customer of the referenced Order Form.

+ +

2.3 Affiliates. The parties agree that their respective Affiliates may also conduct business under this Agreement by entering into Order Forms, which in some cases may be subject to such additional and/or alternative terms and conditions to those contained in this Agreement as may be mutually agreed in the Order Form or an attachment thereto, as applicable. Accordingly, where Affiliates of the parties conduct business hereunder, references to Customer herein shall include any applicable Customer Affiliate, and references to ClickHouse herein shall include any applicable ClickHouse Affiliate. The parties agree that where either of them or one of their Affiliates enters into an Order Form with an Affiliate of the other party, that such Affiliate shall be solely responsible for performing all of its obligations under this Agreement in connection with such Order Form.

+ +

2.4 Performance of Support Services. Subject to Customer’s payment of all fees (as set forth in an applicable Order Form), ClickHouse will provide Customer with Support Services for the Software during an applicable Subscription Term in accordance with this Agreement and the Support Services Policy. Customer will reasonably cooperate with ClickHouse in connection with the Support Services, including, without limitation, by providing ClickHouse reasonable remote access to its installations, server cloud (or hosting provider), Software and equipment in connection therewith. Further, Customer will designate appropriately skilled personnel to serve as ClickHouse’s central contacts in connection with the use, operation and support of the Software. Customer understands that ClickHouse’s performance of Support Services is dependent in part on Customer’s cooperation, actions, and performance. ClickHouse shall not be responsible for any delays or interruptions in its performance of Support Services, or any claims arising therefrom, due to Customer’s lack of cooperation or acts or omissions. ClickHouse may use its Affiliates or subcontractors to provide Support Services to Customer, provided that ClickHouse remains responsible to Customer for performance.

+ +

3. PAYMENT AND TAXES

+ +

3.1 Payment. ClickHouse will invoice Customer for the fees due under each Order Form or otherwise under this Agreement, and Customer will pay such fees within thirty (30) days after receipt of an applicable invoice. All invoices will be paid in the currency set forth on the applicable Order Form. Payments will be made without right of set-off or chargeback. Except as otherwise expressly provided in this Agreement, any and all payments made by Customer pursuant to this Agreement or any Order Form are non-refundable, and all commitments to make any payments hereunder or under any Order Form are non-cancellable.

+ +

3.2 Taxes. All fees stated on an Order Form are exclusive of any applicable sales, use, value added and excise taxes levied upon the delivery or use of the taxable components, if any, of any Subscription purchased by Customer under this Agreement (collectively, “Taxes”). Taxes do not include any taxes on the net income of ClickHouse or any of its Affiliates. Unless Customer provides ClickHouse a valid state sales/use/excise tax exemption certificate or Direct Pay Permit, and provided that ClickHouse separately states any such taxes in the applicable invoice, Customer will pay and be solely responsible for all Taxes. If Customer is required by any foreign governmental authority to deduct or withhold any portion of the amount invoiced for the delivery or use of Support Services under this Agreement, Customer shall increase the sum paid to ClickHouse by an amount necessary for the total payment to ClickHouse equal to the amount originally invoiced.

+ +

4. CONFIDENTIAL INFORMATION

+ +

4.1 Confidential Information. Both parties acknowledge that, in the course of performing this Agreement, they may obtain information relating to products (such as goods, services, and software) of the other party, or relating to the parties themselves, which is of a confidential and proprietary nature ("Confidential Information"). Confidential Information includes materials and all communications concerning ClickHouse's or Customer's business and marketing strategies, including but not limited to employee and customer lists, customer profiles, project plans, design documents, product strategies and pricing data, research, advertising plans, leads and sources of supply, development activities, design and coding, interfaces with the Products, anything provided by either party to the other in connection with the Products and/or Support Services provided under this Agreement, including, without limitation, computer programs, technical drawings, algorithms, know-how, formulas, processes, ideas, inventions (whether patentable or not), schematics and other technical plans and other information of the parties which by its nature can be reasonably expected to be proprietary and confidential, whether it is presented in oral, printed, written, graphic or photographic or other tangible form (including information received, stored or transmitted electronically) even though specific designation as Confidential Information has not been made. Confidential Information also includes any notes, summaries, analyses of the foregoing that are prepared by the receiving party.

+ +

4.2 Non-use and Non-disclosure. The parties shall at all times, both during the Term and thereafter keep in trust and confidence all Confidential Information of the other party using commercially reasonable care (but in no event less than the same degree of care that the receiving party uses to protect its own Confidential Information) and shall not use such Confidential Information other than as necessary to carry out its duties under this Agreement, nor shall either party disclose any such Confidential Information to third parties other than to Affiliates or as necessary to carry out its duties under this Agreement without the other party's prior written consent, provided that each party shall be allowed to disclose Confidential Information of the other party to the extent that such disclosure is approved in writing by such other party, or necessary to enforce its rights under this Agreement.

+ +

4.3 Non-Applicability. The obligations of confidentiality shall not apply to information which (i) has entered the public domain or is otherwise publicly available, except where such entry or availability is the result of a party's breach of this Agreement; (ii) prior to disclosure hereunder was already in the receiving party's possession without restriction as evidenced by appropriate documentation; (iii) subsequent to disclosure hereunder is obtained by the receiving party on a non-confidential basis from a third party who has the right to disclose such information; or (iv) was developed by the receiving party without any use of any of the Confidential Information as evidenced by appropriate documentation.

+ +

4.4 Terms of this Agreement. Except as required by law or governmental regulation, neither party shall disclose, advertise, or publish the terms and conditions of this Agreement without the prior written consent of the other party, except that either party may disclose the terms of this Agreement to potential acquirers, referral partners involved in an applicable transaction, accountants, attorneys and Affiliates pursuant to the terms of a non-disclosure or confidentiality agreement. If Customer is using a third party provider to host a Product, then such provider may also receive, subject to a confidentiality obligation, information related to the terms of this Agreement or Customer’s usage of the applicable Product.

+ +

4.5 Disclosure Required by Law. Notwithstanding anything to the contrary herein, each party may disclose the other party's Confidential Information in order to comply with applicable law and/or an order from a court or other governmental body of competent jurisdiction, and, in connection with compliance with such an order only, if such party: (i) unless prohibited by law, gives the other party prior written notice to such disclosure if the time between that order and such disclosure reasonably permits or, if time does not permit, gives the other party written notice of such disclosure promptly after complying with that order and (ii) fully cooperates with the other party, at the other party's cost and expense, in seeking a protective order, or confidential treatment, or taking other measures to oppose or limit such disclosure. Each party must not release any more of the other party's Confidential Information than is, in the opinion of its counsel, reasonably necessary to comply with an applicable order.

+ +

5. WARRANTIES AND DISCLAIMER OF WARRANTIES

+ +

5.1 Limited Support Services Performance Warranty. ClickHouse warrants that it will perform the Support Services in a professional, workmanlike manner, consistent with generally accepted industry practice, and in accordance with the Support Services Policy. In the event of a breach of the foregoing warranty, ClickHouse’s sole obligation, and Customer’s exclusive remedy, shall be for ClickHouse to re-perform the applicable Support Services.

+ +

5.2 Warranty Disclaimer. EXCEPT AS SET FORTH IN SECTION 5.1 ABOVE, THE SUPPORT SERVICES ARE PROVIDED “AS IS” WITHOUT WARRANTY OF ANY KIND AND CLICKHOUSE MAKES NO ADDITIONAL WARRANTIES, WHETHER EXPRESSED, IMPLIED OR STATUTORY, REGARDING OR RELATING TO THE SUPPORT SERVICES OR ANY MATERIALS FURNISHED OR PROVIDED TO CUSTOMER UNDER THIS AGREEMENT. TO THE MAXIMUM EXTENT PERMITTED UNDER APPLICABLE LAW, CLICKHOUSE SPECIFICALLY DISCLAIMS ALL IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT WITH RESPECT TO THE SUPPORT SERVICES AND ANY MATERIALS FURNISHED OR PROVIDED TO CUSTOMER UNDER THIS AGREEMENT. CUSTOMER UNDERSTANDS AND AGREES THAT THE SUPPORT SERVICES AND ANY MATERIALS FURNISHED OR PROVIDED TO CUSTOMER UNDER THIS AGREEMENT ARE NOT DESIGNED OR INTENDED FOR USE IN THE OPERATION OF NUCLEAR FACILITIES, AIRCRAFT, WEAPONS SYSTEMS, OR LIFE SUPPORT SYSTEMS.

+ +

6. LIMITATION OF LIABILITY

+ +

6.1 Excluded Damages. IN NO EVENT SHALL CUSTOMER OR CLICKHOUSE, OR THEIR RESPECTIVE AFFILIATES, BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF USE, BUSINESS INTERRUPTION, LOSS OF DATA, COST OF SUBSTITUTE GOODS OR SERVICES, OR FOR ANY PUNITIVE, INDIRECT, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES OF ANY KIND IN CONNECTION WITH OR ARISING OUT OF THE PERFORMANCE OF OR FAILURE TO PERFORM THIS AGREEMENT, WHETHER ALLEGED AS A BREACH OF CONTRACT OR TORTIOUS CONDUCT, INCLUDING NEGLIGENCE, EVEN IF A PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.

+ +

6.2 Damages Cap. EXCEPT WITH RESPECT TO (I) A PARTY’S BREACH OF ITS OBLIGATIONS UNDER SECTION 4, (II) AMOUNTS PAYABLE BY CUSTOMER UNDER SECTION 3 OF THIS AGREEMENT AND EACH ORDER FORM, AND (III) CUSTOMER'S VIOLATIONS OF THE USE RESTRICTIONS SET FORTH IN THIS AGREEMENT, IN NO EVENT SHALL CLICKHOUSE'S OR CUSTOMER’S TOTAL, CUMULATIVE LIABILITY UNDER ANY ORDER FORM EXCEED THE AMOUNT PAID OR PAYABLE BY CUSTOMER TO CLICKHOUSE UNDER THIS AGREEMENT FOR THE AFFECTED SUPPORT SERVICES DELIVERED AND/OR MADE AVAILABLE TO CUSTOMER UNDER SUCH ORDER FORM FOR THE TWELVE (12) MONTH PERIOD IMMEDIATELY PRIOR TO THE FIRST EVENT GIVING RISE TO LIABILITY.

+ +

6.3 Basis of the Bargain. THE ALLOCATIONS OF LIABILITY IN THIS SECTION 6 REPRESENT THE AGREED AND BARGAINED FOR UNDERSTANDING OF THE PARTIES, AND THE COMPENSATION OF CLICKHOUSE FOR THE SUPPORT SERVICES PROVIDED HEREUNDER REFLECTS SUCH ALLOCATIONS. THE FOREGOING LIMITATIONS, EXCLUSIONS AND DISCLAIMERS WILL APPLY TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, EVEN IF ANY REMEDY FAILS IN ITS ESSENTIAL PURPOSE.

+ +

7. TERM AND TERMINATION

+ +

7.1 Subscription Term. The initial Subscription Term for each Subscription will commence and expire in accordance with the start date and end date set forth on the applicable Order Form, unless earlier terminated in in accordance with Section 7.3 below. Thereafter, each Subscription may be renewed for additional one (1) year periods upon the mutual written agreement of the parties. The initial Subscription Term, plus any subsequent renewal Subscription Term shall be the "Subscription Term".

+ +

7.2 Agreement Term. This Agreement will commence on the Effective Date and, unless earlier terminated in accordance with Section 7.3(b) below, continue in force and effect for a period of two (2) years. Thereafter, the term of this Agreement shall automatically renew for additional one (1) year periods unless either party gives written notice to the other of its intention not to renew the Agreement at least thirty (30) days prior to the expiration of the then-current term. The initial term of this Agreement, plus any subsequent renewal term, shall be the "Term" of this Agreement. Notwithstanding any expiration of this Agreement, its terms will continue to apply to any Subscription that has not been terminated or for which the Subscription Term has not expired.

+ +

7.3 Termination.

+
    +
  1. Subscriptions. Each party may terminate a Subscription upon giving notice in writing to the other party if the non-terminating party commits a material breach of this Agreement with respect to such Subscription, and has failed to cure such breach within thirty (30) days following a request in writing from the notifying party to do so. Upon the termination or expiration of a Subscription, the rights and obligations of the parties with respect thereto will, subject to Section 7.4 below, cease, provided that termination of a Subscription under this subsection (a) will not result in termination of any other Subscriptions.
    2. +
  2. Agreement. Either party may terminate this Agreement upon giving notice in writing to the other party if the non-terminating party commits a material breach of this Agreement with respect to any active Subscriptions hereunder, and has failed to cure such breach within thirty (30) days following a request in writing from the notifying party to do so. For the avoidance of doubt, termination of this Agreement under this subsection (b) will result in the termination of all Subscriptions and Order Forms.
    3. +
+ +

7.4 Survival. Upon the expiration or termination of an Order Form or this Agreement, (i) Customer shall have no further rights under any affected Subscription(s); and (ii) any payment obligations accrued under Section 3, as well as the provisions of Sections 1, 4, 5, 6, 7, 7.4 and 9 of this Agreement will survive such expiration or termination.

+ +

8. GENERAL

+ +

8.1 Anti-Corruption. Each party acknowledges that it is aware of, understands and has complied and will comply with, all applicable U.S. and foreign anti-corruption laws, including without limitation, the U.S. Foreign Corrupt Practices Act of 1977 and the U.K. Bribery Act of 2010, and similarly applicable anti-corruption and anti-bribery laws ("Anti-Corruption Laws"). Each party agrees that no one acting on its behalf will give, offer, agree or promise to give, or authorize the giving directly or indirectly, of any money or other thing of value, including travel, entertainment, or gifts, to anyone as an unlawful inducement or reward for favorable action or forbearance from action or the exercise of unlawful influence (a) to any governmental official or employee (including employees of government-owned and government-controlled corporations or agencies or public international organizations), (b) to any political party, official of a political party, or candidate, (c) to an intermediary for payment to any of the foregoing, or (d) to any other person or entity in a corrupt or improper effort to obtain or retain business or any commercial advantage, such as receiving a permit or license, or directing business to any person. Improper payments, provisions, bribes, kickbacks, influence payments, or other unlawful provisions to any person are prohibited under this Agreement.

+ +

8.2 Assignment. Neither party may assign this Agreement, in whole or in part, without the prior written consent of the other party, provided that no such consent will be required to assign this Agreement in its entirety to (i) an Affiliate that is able to satisfy the obligations of the assigning party under this Agreement or (ii) a successor in interest in connection with a merger, acquisition or sale of all or substantially all of the assigning party's assets. Any assignment in violation of this Section shall be void, ab initio, and of no effect. Subject to the foregoing, this Agreement is binding upon, inures to the benefit of and is enforceable by, the parties and their respective permitted successors and assigns.

+ +

8.3 Attorneys' Fees. If any action or proceeding, whether regulatory, administrative, at law or in equity is commenced or instituted to enforce or interpret any of the terms or provisions of this Agreement, the prevailing party in any such action or proceeding shall be entitled to recover its reasonable attorneys' fees, expert witness fees, costs of suit and expenses, in addition to any other relief to which such prevailing party may be entitled. As used herein, "prevailing party" includes without limitation, a party who dismisses an action for recovery hereunder in exchange for payment of the sums allegedly due, performance of covenants allegedly breached, or consideration substantially equal to the relief sought in the action.

+ +

8.4 California Consumer Privacy Act (CCPA). ClickHouse is a “Service Provider” as such term is defined under §1798.140(v) of the CCPA. As such ClickHouse shall not retain, use or disclose any personal information (as defined in the CCPA) received from Customer during the Term of this Agreement for any purpose other than the specific purpose of providing the Support Services specified in this Agreement or for such other business purpose as is specified in this Agreement.

+ +

8.5 Customer Identification. ClickHouse may identify Customer as a user of the Support Services, on its website, through a press release issued by ClickHouse and in other promotional materials.

+ +

8.6 Feedback. Customer, Customer’s Affiliates, and their respective agents, may volunteer feedback to ClickHouse, and/or its Affiliates, about the Support Services (“Feedback”). ClickHouse and its Affiliates shall be irrevocably entitled to use that Feedback, for any purpose and without any duty to account. provided that, in doing so, they may not breach their obligations of confidentiality under Section 4 of this Agreement.

+ +

8.7 Force Majeure. Except with respect to payment obligations, neither party will be liable for, or be considered to be in breach of, or in default under, this Agreement, as a result of any cause or condition beyond such party's reasonable control.

+ +

8.8 Governing Law, Jurisdiction and Venue.

+ +
    +
  1. Customers in California. If Customer is located in California (as determined by the Customer address on the applicable Order Form), this Agreement will be governed by the laws of the State of California, without regard to its conflict of laws principles, and all suits hereunder will be brought solely in Federal Court for the Northern District of California, or if that court lacks subject matter jurisdiction, in any California State Court located in Santa Clara County.
    2. +
  2. Customers Outside of California. If Customer is located anywhere other than California (as determined by the Customer address on the applicable Order Form), this Agreement will be governed by the laws of the State of Delaware, without regard to its conflict of laws principles, and all suits hereunder will be brought solely in Federal Court for the District of Delaware, or if that court lacks subject matter jurisdiction, in any Delaware State Court located in Wilmington, Delaware.
    3. + +
  3. All Customers. This Agreement shall not be governed by the 1980 UN Convention on Contracts for the International Sale of Goods. The parties hereby irrevocably waive any and all claims and defenses either might otherwise have in any action or proceeding in any of the applicable courts set forth in (a) or (b) above, based upon any alleged lack of personal jurisdiction, improper venue, forum non conveniens, or any similar claim or defense.
    4. + +
  4. Equitable Relief. A breach or threatened breach, by either party of Section 4 may cause irreparable harm for which damages at law may not provide adequate relief, and therefore the non-breaching party shall be entitled to seek injunctive relief without being required to post a bond.
    5. + +
+ +

8.9 Non-waiver. Any failure of either party to insist upon or enforce performance by the other party of any of the provisions of this Agreement or to exercise any rights or remedies under this Agreement will not be interpreted or construed as a waiver or relinquishment of such party's right to assert or rely upon such provision, right or remedy in that or any other instance.

+ +

8.10 Notices. Any notice or other communication under this Agreement given by either party to the other will be deemed to be properly given if given in writing and delivered in person or by e-mail, if acknowledged received by return e-mail or followed within one day by a delivered or mailed copy of such notice, or if mailed, properly addressed and stamped with the required postage, to the intended recipient at its address specified on an Order Form. Notices to ClickHouse may also be sent to legal@ClickHouse.com. Either party may from time to time change its address for notices under this Section by giving the other party notice of the change in accordance with this Section.

+ +

8.11 Relationship of the Parties. The relationship of the parties hereunder shall be that of independent contractors, and nothing herein shall be deemed or construed to create any employment, agency or fiduciary relationship between the parties. Each party shall be solely responsible for the supervision, direction, control and payment of its personnel, including, without limitation, for taxes, deductions and withholdings, compensation and benefits, and nothing herein will be deemed to result in either party having an employer-employee relationship with the personnel of the other party.

+ +

8.12 Severability. If any provision of this Agreement is held to be invalid or unenforceable, the remaining portions will remain in full force and effect and such provision will be enforced to the maximum extent possible so as to give effect the intent of the parties and will be reformed to the extent necessary to make such provision valid and enforceable.

+ +

8.13 Entire Agreement; Amendment. This Agreement, together with any Order Forms executed by the parties, and the Support Services Policy, each of which is hereby incorporated herein by this reference, constitutes the entire agreement between the parties concerning the subject matter hereof, and it supersedes, and its terms govern, all prior proposals, agreements, or other communications between the parties, oral or written, regarding such subject matter. This Agreement may be executed in any number of counterparts, each of which when so executed and delivered shall be deemed an original, and all of which together shall constitute one and the same agreement. Execution of a scanned copy will have the same force and effect as execution of an original, and a scanned signature will be deemed an original and valid signature. In the event of any conflict between the terms and conditions of any of the foregoing documents, the conflict shall be resolved based on the following order of precedence: (i) an applicable Order Form (but only for the transaction thereunder), (ii) an applicable Addendum (including any exhibits, attachments and addenda thereto), (iii) this Agreement, and (iv) the Support Services Policy. For the avoidance of doubt, the parties hereby expressly acknowledge and agree that if Customer issues any purchase orders or similar documents in connection with its purchase of a Subscription, it shall do so only for the purpose of Section 2.2(2) or for its own internal, administrative purposes and not with the intent to provide any contractual terms. By entering into this Agreement, whether prior to or following receipt of Customer's purchase order or any similar document, the parties are hereby expressly showing their intention not to be contractually bound by the contents of any such purchase order or similar document, which are hereby deemed rejected and extraneous to this Agreement, and ClickHouse's performance of this Agreement shall not amount to: (i) an acceptance by conduct of any terms set out or referred to in the purchase order or similar document; (ii) an amendment of this Agreement, nor (iii) an agreement to amend this Agreement. This Agreement shall not be modified except by a subsequently dated, written amendment that expressly amends this Agreement and which is signed on behalf of ClickHouse and Customer by their duly authorized representatives. The parties agree that the terms and conditions of this Agreement are a result of mutual negotiations. Therefore, the rule of construction that any ambiguity shall apply against the drafter is not applicable and will not apply to this Agreement. Any ambiguity shall be reasonably construed as to its fair meaning and not strictly for or against one party regardless of who authored the ambiguous language.

+ + +
+
\ No newline at end of file diff --git a/website/templates/support/agreement-hero.html b/website/templates/support/agreement-hero.html new file mode 100644 index 00000000000..ea97fb7729a --- /dev/null +++ b/website/templates/support/agreement-hero.html @@ -0,0 +1,10 @@ +
+
+
+ +

+ {{ _('Clickhouse, Inc.
Subscription Agreement') }} +

+ +
+
\ No newline at end of file diff --git a/website/templates/support/policy-content.html b/website/templates/support/policy-content.html new file mode 100644 index 00000000000..d6efae2fb67 --- /dev/null +++ b/website/templates/support/policy-content.html @@ -0,0 +1,55 @@ +
+
+

Effective Date: January 28, 2022

+ +

This ClickHouse Support Services Policy sets forth ClickHouse’s policy for the provision of Support Services to its Subscription customers (each a “Customer”). Capitalized terms used but not defined herein, have the definition set forth in the applicable Subscription Agreement between ClickHouse and Customer (the “Agreement”).

+ +
    +
  1. Defined Terms +
      +
    1. "Business Day" means Monday through Friday other than a day designated from time to time as a national holiday in the place from which Support Services may be provided.
      2. +
    2. “Long-Term Supported Release” means the official release for the Software which has been designated as a release for long term support. ClickHouse designates its as Long-Term Supported Releases by including the letters“lts” in the release number, which are typically March and August releases.
      3. +
    3. “Regular Stable Sequential Release” means the official software release for the Software which has not been designated as a Long Term Supported Release.
      4. +
    4. “Support” means technical support by telephone, chat, or email provided by ClickHouse to a Support Contact concerning a Support Incident.
      5. +
    5. “Support Contact” means a single named individual who is authorized to contact ClickHouse to use the Support Services.
      6. +
    6. "Support Incident" means a single issue or error with the Software that is raised with ClickHouse by a Support Contact. Each Support Incident will be assigned a unique ID by ClickHouse. In the situation where multiple similar or equivalent cases are opened for a single Support Incident, ClickHouse may choose to consolidate these into a single support case, in which case it shall promptly notify Customer.
      7. +
    +
    2. +
  2. Scope and Performance of Support Services. +

    The scope of the Support Services provided to Customer includes general assistance and support regarding the installation of the Software and basic technical configuration of the Software, as well as operational or development assistance on how to use the Software. ClickHouse shall use commercially reasonable efforts to meet the applicable target response times set forth in Section 3 below. Customer acknowledges that the time required for resolution of issues may vary depending on the specific circumstances of each problem, including, without limitation, the nature of the incident/problem, the extent and accuracy of information available about the incident/problem, and the level of Customer's cooperation and responsiveness in providing materials, information, access and support reasonably required by ClickHouse to achieve problem resolution.

    +
      +
    • Normal Business Hours: +
        +
      • 24x7x365 for Severity 1 Issues
        • +
      • Monday 08:00 CEST through Friday 17:00 US Pacific time for Severity 2 and Severity 3 issues
        • +
      +
      • +
    • Number of Support Contacts: unlimited
      • +
    • Emergency Patches: yes
      • +
    • Annual Support Incidents: unlimited
      • +
    +
    3. +
  3. Severity Levels and ClickHouse Efforts. +

    Severity Level 1

    +

    A Severity Level 1 issue is a major production error within the software that severely impacts the Customer's use of the software for production purposes, such as the loss of production data or where production systems are not functioning and no work-around exists. ClickHouse will use continuous efforts during applicable Normal Business Hours to provide a resolution for any Level 1 issues as soon as is commercially reasonable.

    +

    Severity Level 2

    +

    A Severity Level 2 issue is an error within the software where the customer's system is functioning for production purposes but in a reduced capacity, such as a problem that is causing significant impact to portions of the customer's business operations and productivity, or where the software is exposed to potential loss or interruption of service. ClickHouse will use continuous efforts during the Normal Business Hours to provide a resolution for any Severity Level 2 issues.

    +

    Severity Level 3

    +

    A Severity Level 3 issue is a medium-to-low impact error that involves partial and/or non-critical loss of functionality for production purposes or development purposes, such as a problem that impairs some operations but allows the customer's operations to continue to function. Errors for which there is limited or no loss or functionality or impact to the customer's operation and for which there is an easy work-around qualify as Severity Level 3. General questions are also Severity Level issues. ClickHouse will use reasonable efforts to provide a resolution for any Severity Level 3 issues in time for an upcoming release of the software. All inbound production email cases shall have an initial status of Severity Level 3.

    +
    4. +
  4. Customer Obligations. +

    Customer agrees to provide ClickHouse with reasonable: (i) detail of the nature of and circumstances surrounding the issue, (ii) access to Customer's environment as necessary to enable ClickHouse to provide Support Services; and (iii) cooperation in the diagnosis and resolution of any issues.

    +
    5. +
  5. Supported Versions. +

    Notwithstanding anything else, ClickHouse will support the current Regular Stable Release in conjunction with the two (2) prior Regular Stable Releases of the Software or a minimum period of three (3) months from the date of the current Regular Stable Release. ClickHouse will support the current Long-Term Supported Release in conjunction with the one (1) prior Long-Term Supported Release of the Software or a minimum period of one (1) year from the date of the current Long-Term Supported Release.

    + +
    6. +
  6. Support Service Exclusions. +

    ClickHouse will have no obligation to provide Support Services to Customer in the event that (i) the Software has been changed, modified or damaged by Customer or anyone other than ClickHouse, (ii) the problem is caused by Customer's negligence, misconduct, or misuse of the Software, a hardware malfunction, or other causes beyond the reasonable control of ClickHouse, (iii) the problem is due to third party software, (iv) the Software is being hosted by a third party that is offering the Software as a service (v) Customer has not installed or implemented any Software releases made generally available or is not running a then supported version of the Software as provided by ClickHouse or (vi) information requested by Customer could reasonably be expected to assist in the development, deployment, enablement and/or maintenance of any non-ClickHouse software that competes with ClickHouse's commercial software products. The Support Services do not cover the support of any third party software which integrates with the Software or the investigation into a potential or actual security incident in a Customer environment, including but not limited to the analysis and response to security events and signals. In addition, the Support Services do not include the following: (a) use of any version of a Software that is not designated as a production release (such as a milestone or release candidate or code contained in the sandbox or any other repository that is not packaged into a production release distribution); (b) Customer's failure to comply with operating instructions contained in the documentation for the Software; (c) installation, configuration, management and operation of Customer's applications; (d) APIs, interfaces or data formats other than those included with the Software; or (e) any training.

    +
    7. + +
+
+
\ No newline at end of file diff --git a/website/templates/support/policy-hero.html b/website/templates/support/policy-hero.html new file mode 100644 index 00000000000..dc551dc434c --- /dev/null +++ b/website/templates/support/policy-hero.html @@ -0,0 +1,10 @@ +
+
+
+ +

+ {{ _('ClickHouse Support Services Policy') }} +

+ +
+
\ No newline at end of file