thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into mvcc_prototype

Browse Source
This commit is contained in:
Alexander Tokmakov 2022-02-03 18:56:44 +03:00
commit ca5f951558
155 changed files with 1886 additions and 784 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/master.yml vendored
View File

@ -86,6 +86,7 @@ jobs:
StyleCheck:
needs: DockerHubPush
runs-on: [self-hosted, style-checker]
if: ${{ success() || failure() }}
steps:
- name: Set envs
run: |
@ -93,6 +94,8 @@ jobs:
TEMP_PATH=${{ runner.temp }}/style_check
EOF
- name: Download changed images
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job
continue-on-error: true
uses: actions/download-artifact@v2
with:
name: changed_images

3
.github/workflows/pull_request.yml vendored
View File

@ -111,6 +111,7 @@ jobs:
StyleCheck:
needs: DockerHubPush
runs-on: [self-hosted, style-checker]
if: ${{ success() || failure() }}
steps:
- name: Set envs
run: |
@ -118,6 +119,8 @@ jobs:
TEMP_PATH=${{ runner.temp }}/style_check
EOF
- name: Download changed images
# even if artifact does not exist, e.g. on `do not test` label or failed Docker job
continue-on-error: true
uses: actions/download-artifact@v2
with:
name: changed_images

38
.github/workflows/tags_stable.yml vendored Normal file
View File

@ -0,0 +1,38 @@
name: TagsStableWorkflow
# - Gets artifacts from S3
# - Sends it to JFROG Artifactory
# - Adds them to the release assets
on: # yamllint disable-line rule:truthy
push:
tags:
- 'v*-stable'
- 'v*-lts'
jobs:
UpdateVersions:
runs-on: [self-hosted, style-checker]
steps:
- name: Get tag name
run: echo "GITHUB_TAG=${GITHUB_REF#refs/tags/}" >> "$GITHUB_ENV"
- name: Check out repository code
uses: actions/checkout@v2
with:
ref: master
- name: Generate versions
run: |
git fetch --tags
./utils/list-versions/list-versions.sh > ./utils/list-versions/version_date.tsv
- name: Create Pull Request
uses: peter-evans/create-pull-request@v3
with:
commit-message: Update version_date.tsv after ${{ env.GITHUB_TAG }}
branch: auto/${{ env.GITHUB_TAG }}
delete-branch: true
title: Update version_date.tsv after ${{ env.GITHUB_TAG }}
body: |
Update version_date.tsv after ${{ env.GITHUB_TAG }}
Changelog category (leave one):
- Not for changelog (changelog entry is not required)

2
CMakeLists.txt
View File

@ -67,7 +67,7 @@ if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/.git" AND NOT EXISTS "${ClickHouse_SOURC
message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive")
endif ()
include (cmake/find/ccache.cmake)
include (cmake/ccache.cmake)
# Take care to add prlimit in command line before ccache, or else ccache thinks that
# prlimit is compiler, and clang++ is its input file, and refuses to work with

3
SECURITY.md
View File

@ -22,9 +22,10 @@ The following versions of ClickHouse server are currently being supported with s
| 21.7 | :x: |
| 21.8 | ✅ |
| 21.9 | :x: |
| 21.10 | ✅ |
| 21.10 | :x: |
| 21.11 | ✅ |
| 21.12 | ✅ |
| 22.1 | ✅ |
## Reporting a Vulnerability

2
base/base/LineReader.cpp
View File

@ -81,7 +81,7 @@ replxx::Replxx::completions_t LineReader::Suggest::getCompletions(const String &
std::lock_guard lock(mutex);
/// Only perform case sensitive completion when the prefix string contains any uppercase characters
if (std::none_of(prefix.begin(), prefix.end(), [&](auto c) { return c >= 'A' && c <= 'Z'; }))
if (std::none_of(prefix.begin(), prefix.end(), [](char32_t x) { return iswupper(static_cast<wint_t>(x)); }))
range = std::equal_range(
words_no_case.begin(), words_no_case.end(), last_word, [prefix_length](std::string_view s, std::string_view prefix_searched)
{

30
base/base/ReplxxLineReader.cpp
View File

@ -25,13 +25,6 @@ void trim(String & s)
s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }).base(), s.end());
}
/// Check if string ends with given character after skipping whitespaces.
bool ends_with(const std::string_view & s, const std::string_view & p)
{
auto ss = std::string_view(s.data(), s.rend() - std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) { return !std::isspace(ch); }));
return ss.ends_with(p);
}
std::string getEditor()
{
const char * editor = std::getenv("EDITOR");
@ -132,6 +125,12 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx)
}
static bool replxx_last_is_delimiter = false;
void ReplxxLineReader::setLastIsDelimiter(bool flag)
{
replxx_last_is_delimiter = flag;
}
ReplxxLineReader::ReplxxLineReader(
Suggest & suggest,
const String & history_file_path_,
@ -185,6 +184,7 @@ ReplxxLineReader::ReplxxLineReader(
rx.set_completion_callback(callback);
rx.set_complete_on_empty(false);
rx.set_word_break_characters(word_break_characters);
rx.set_ignore_case(true);
if (highlighter)
rx.set_highlighter_callback(highlighter);
@ -196,21 +196,11 @@ ReplxxLineReader::ReplxxLineReader(
auto commit_action = [this](char32_t code)
{
std::string_view str = rx.get_state().text();
/// Always commit line when we see extender at the end. It will start a new prompt.
for (const auto * extender : extenders)
if (ends_with(str, extender))
return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
/// If we see an delimiter at the end, commit right away.
for (const auto * delimiter : delimiters)
if (ends_with(str, delimiter))
return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
/// If we allow multiline and there is already something in the input, start a newline.
if (multiline && !input.empty())
/// NOTE: Lexer is only available if we use highlighter.
if (highlighter && multiline && !replxx_last_is_delimiter)
return rx.invoke(Replxx::ACTION::NEW_LINE, code);
replxx_last_is_delimiter = false;
return rx.invoke(Replxx::ACTION::COMMIT_LINE, code);
};
/// bind C-j to ENTER action.

3
base/base/ReplxxLineReader.h
View File

@ -19,6 +19,9 @@ public:
void enableBracketedPaste() override;
/// If highlight is on, we will set a flag to denote whether the last token is a delimiter.
/// This is useful to determine the behavior of <ENTER> key when multiline is enabled.
static void setLastIsDelimiter(bool flag);
private:
InputStatus readOneLine(const String & prompt) override;
void addToHistory(const String & line) override;

2
base/base/logger_useful.h
View File

@ -12,6 +12,8 @@ namespace
{
template <typename... Ts> constexpr size_t numArgs(Ts &&...) { return sizeof...(Ts); }
template <typename T, typename... Ts> constexpr auto firstArg(T && x, Ts &&...) { return std::forward<T>(x); }
/// For implicit conversion of fmt::basic_runtime<> to char* for std::string ctor
template <typename T, typename... Ts> constexpr auto firstArg(fmt::basic_runtime<T> && data, Ts &&...) { return data.str.data(); }
}

6
base/daemon/BaseDaemon.cpp
View File

@ -317,7 +317,7 @@ private:
else
error_message = "Sanitizer trap.";
LOG_FATAL(log, error_message);
LOG_FATAL(log, fmt::runtime(error_message));
if (stack_trace.getSize())
{
@ -330,11 +330,11 @@ private:
for (size_t i = stack_trace.getOffset(); i < stack_trace.getSize(); ++i)
bare_stacktrace << ' ' << stack_trace.getFramePointers()[i];
LOG_FATAL(log, bare_stacktrace.str());
LOG_FATAL(log, fmt::runtime(bare_stacktrace.str()));
}
/// Write symbolized stack trace line by line for better grep-ability.
stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, s); });
stack_trace.toStringEveryLine([&](const std::string & s) { LOG_FATAL(log, fmt::runtime(s)); });
#if defined(OS_LINUX)
/// Write information about binary checksum. It can be difficult to calculate, so do it only after printing stack trace.

0
cmake/find/ccache.cmake → cmake/ccache.cmake
View File

0
cmake/find/cxx.cmake → cmake/cxx.cmake
View File

2
cmake/darwin/default_libs.cmake
View File

@ -22,7 +22,7 @@ set(CMAKE_OSX_DEPLOYMENT_TARGET 10.15)
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
include (cmake/find/cxx.cmake)
include (cmake/cxx.cmake)
target_link_libraries(global-group INTERFACE
$<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>

4
cmake/freebsd/default_libs.cmake
View File

@ -22,8 +22,8 @@ set(CMAKE_C_STANDARD_LIBRARIES ${DEFAULT_LIBS})
set(THREADS_PREFER_PTHREAD_FLAG ON)
find_package(Threads REQUIRED)
include (cmake/find/unwind.cmake)
include (cmake/find/cxx.cmake)
include (cmake/unwind.cmake)
include (cmake/cxx.cmake)
target_link_libraries(global-group INTERFACE
$<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>

4
cmake/linux/default_libs.cmake
View File

@ -42,8 +42,8 @@ if (NOT OS_ANDROID)
add_subdirectory(base/harmful)
endif ()
include (cmake/find/unwind.cmake)
include (cmake/find/cxx.cmake)
include (cmake/unwind.cmake)
include (cmake/cxx.cmake)
target_link_libraries(global-group INTERFACE
-Wl,--start-group

0
cmake/find/unwind.cmake → cmake/unwind.cmake
View File

65
contrib/arrow-cmake/CMakeLists.txt
View File

@ -29,12 +29,6 @@ if (OS_FREEBSD)
message (FATAL_ERROR "Using internal parquet library on FreeBSD is not supported")
endif()
if(USE_STATIC_LIBRARIES)
set(FLATBUFFERS_LIBRARY flatbuffers)
else()
set(FLATBUFFERS_LIBRARY flatbuffers_shared)
endif()
set (CMAKE_CXX_STANDARD 17)
set(ARROW_VERSION "6.0.1")
@ -95,9 +89,16 @@ set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "Skip flatbuffers tests")
add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}")
message(STATUS "FLATBUFFERS_LIBRARY: ${FLATBUFFERS_LIBRARY}")
add_library(_flatbuffers INTERFACE)
if(USE_STATIC_LIBRARIES)
target_link_libraries(_flatbuffers INTERFACE flatbuffers)
else()
target_link_libraries(_flatbuffers INTERFACE flatbuffers_shared)
endif()
target_include_directories(_flatbuffers INTERFACE ${FLATBUFFERS_INCLUDE_DIR})
# === hdfs
# NOTE: cannot use ch_contrib::hdfs since it's INCLUDE_DIRECTORIES does not includes trailing "hdfs/"
set(HDFS_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/")
# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features.
@ -123,8 +124,6 @@ configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/A
# ARROW_ORC + adapters/orc/CMakefiles
set(ORC_SRCS
"${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc"
"${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc"
"${ORC_SOURCE_SRC_DIR}/Exceptions.cc"
"${ORC_SOURCE_SRC_DIR}/OrcFile.cc"
"${ORC_SOURCE_SRC_DIR}/Reader.cc"
@ -151,6 +150,22 @@ set(ORC_SRCS
"${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc"
)
add_library(_orc ${ORC_SRCS})
target_link_libraries(_orc PRIVATE
ch_contrib::protobuf
ch_contrib::lz4
ch_contrib::snappy
ch_contrib::zlib
ch_contrib::zstd)
target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR})
target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR})
target_include_directories(_orc SYSTEM PRIVATE
${ORC_SOURCE_SRC_DIR}
${ORC_SOURCE_WRAP_DIR}
${ORC_BUILD_SRC_DIR}
${ORC_ADDITION_SOURCE_DIR}
${ARROW_SRC_DIR})
# === arrow
@ -336,7 +351,8 @@ set(ARROW_SRCS
"${LIBRARY_DIR}/ipc/reader.cc"
"${LIBRARY_DIR}/ipc/writer.cc"
${ORC_SRCS}
"${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc"
"${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc"
)
add_definitions(-DARROW_WITH_LZ4)
@ -356,30 +372,27 @@ endif ()
add_library(_arrow ${ARROW_SRCS})
# Arrow dependencies
add_dependencies(_arrow ${FLATBUFFERS_LIBRARY})
target_link_libraries(_arrow PRIVATE
boost::filesystem
target_link_libraries(_arrow PRIVATE ${FLATBUFFERS_LIBRARY} boost::filesystem)
_flatbuffers
ch_contrib::double_conversion
ch_contrib::lz4
ch_contrib::snappy
ch_contrib::zlib
ch_contrib::zstd
ch_contrib::zstd
)
target_link_libraries(_arrow PUBLIC _orc)
add_dependencies(_arrow protoc)
target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR})
target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${CMAKE_CURRENT_BINARY_DIR}/cpp/src")
target_link_libraries(_arrow PRIVATE ch_contrib::double_conversion)
target_link_libraries(_arrow PRIVATE ch_contrib::protobuf)
target_link_libraries(_arrow PRIVATE ch_contrib::lz4)
target_link_libraries(_arrow PRIVATE ch_contrib::snappy)
target_link_libraries(_arrow PRIVATE ch_contrib::zlib)
target_link_libraries(_arrow PRIVATE ch_contrib::zstd)
target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR})
target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR})
target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_SRC_DIR})
target_include_directories(_arrow SYSTEM PRIVATE ${ORC_SOURCE_WRAP_DIR})
target_include_directories(_arrow SYSTEM PRIVATE ${ORC_BUILD_SRC_DIR})
target_include_directories(_arrow SYSTEM PRIVATE ${ORC_ADDITION_SOURCE_DIR})
target_include_directories(_arrow SYSTEM PRIVATE ${ARROW_SRC_DIR})
target_include_directories(_arrow SYSTEM PRIVATE ${FLATBUFFERS_INCLUDE_DIR})
target_include_directories(_arrow SYSTEM PRIVATE ${HDFS_INCLUDE_DIR})
# === parquet

2
contrib/fmtlib vendored

@ -1 +1 @@
Subproject commit c108ee1d590089ccf642fc85652b845924067af2
Subproject commit b6f4ceaed0a0a24ccf575fab6c56dd50ccf6f1a9

5
contrib/fmtlib-cmake/CMakeLists.txt
View File

@ -1,7 +1,10 @@
set (SRCS
# NOTE: do not build module for now:
# ../fmtlib/src/fmt.cc
../fmtlib/src/format.cc
../fmtlib/src/os.cc
../fmtlib/include/fmt/args.h
../fmtlib/include/fmt/chrono.h
../fmtlib/include/fmt/color.h
../fmtlib/include/fmt/compile.h
@ -11,9 +14,9 @@ set (SRCS
../fmtlib/include/fmt/locale.h
../fmtlib/include/fmt/os.h
../fmtlib/include/fmt/ostream.h
../fmtlib/include/fmt/posix.h
../fmtlib/include/fmt/printf.h
../fmtlib/include/fmt/ranges.h
../fmtlib/include/fmt/xchar.h
)
add_library(_fmt ${SRCS})

2
contrib/replxx vendored

@ -1 +1 @@
Subproject commit f019cba7ea1bcd1b4feb7826f28ed57fb581b04c
Subproject commit c745b3fb012ee5ae762fbc8cd7a40c4dc3fe15df

5
contrib/rocksdb-cmake/CMakeLists.txt
View File

@ -72,11 +72,6 @@ else()
if(WITH_ZSTD)
add_definitions(-DZSTD)
include_directories(${ZSTD_INCLUDE_DIR})
include_directories("${ZSTD_INCLUDE_DIR}/common")
include_directories("${ZSTD_INCLUDE_DIR}/dictBuilder")
include_directories("${ZSTD_INCLUDE_DIR}/deprecated")
list(APPEND THIRDPARTY_LIBS ch_contrib::zstd)
endif()
endif()

6
docker/test/stateless/run.sh
View File

@ -12,7 +12,11 @@ dpkg -i package_folder/clickhouse-common-static_*.deb
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb
dpkg -i package_folder/clickhouse-server_*.deb
dpkg -i package_folder/clickhouse-client_*.deb
dpkg -i package_folder/clickhouse-test_*.deb
if [[ -n "$TEST_CASES_FROM_DEB" ]] && [[ "$TEST_CASES_FROM_DEB" -eq 1 ]]; then
dpkg -i package_folder/clickhouse-test_*.deb
else
ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
fi
# install test configs
/usr/share/clickhouse-test/config/install.sh

2
docs/_includes/cmake_in_clickhouse_header.md
View File

@ -22,7 +22,7 @@ cmake .. \
1. ClickHouse's source CMake files (located in the root directory and in `/src`).
2. Arch-dependent CMake files (located in `/cmake/*os_name*`).
3. Libraries finders (search for contrib libraries, located in `/cmake/find`).
3. Libraries finders (search for contrib libraries, located in `/contrib/*/CMakeLists.txt`).
3. Contrib build CMake files (used instead of libraries' own CMake files, located in `/cmake/modules`)
## List of CMake flags

2
docs/_includes/install/deb.sh
View File

@ -8,4 +8,4 @@ sudo apt-get update
sudo apt-get install -y clickhouse-server clickhouse-client
sudo service clickhouse-server start
clickhouse-client
clickhouse-client # or "clickhouse-client --password" if you set up a password.

2
docs/_includes/install/rpm.sh
View File

@ -4,4 +4,4 @@ sudo yum-config-manager --add-repo https://repo.clickhouse.com/rpm/clickhouse.re
sudo yum install clickhouse-server clickhouse-client
sudo /etc/init.d/clickhouse-server start
clickhouse-client
clickhouse-client # or "clickhouse-client --password" if you set up a password.

6
docs/en/sql-reference/syntax.md
View File

@ -30,7 +30,7 @@ There may be any number of space symbols between syntactical constructions (incl
ClickHouse supports either SQL-style and C-style comments:
- SQL-style comments start with `--` and continue to the end of the line, a space after `--` can be omitted.
- SQL-style comments start with `--`, `#!` or `# ` and continue to the end of the line, a space after `--` and `#!` can be omitted.
- C-style are from `/*` to `*/`and can be multiline, spaces are not required either.
## Keywords {#syntax-keywords}
@ -106,9 +106,9 @@ In queries, you can check `NULL` using the [IS NULL](../sql-reference/operators/
### Heredoc {#heredeoc}
A [heredoc](https://en.wikipedia.org/wiki/Here_document) is a way to define a string (often multiline), while maintaining the original formatting. A heredoc is defined as a custom string literal, placed between two `$` symbols, for example `$heredoc$`. A value between two heredocs is processed "as-is".
A [heredoc](https://en.wikipedia.org/wiki/Here_document) is a way to define a string (often multiline), while maintaining the original formatting. A heredoc is defined as a custom string literal, placed between two `$` symbols, for example `$heredoc$`. A value between two heredocs is processed "as-is".
You can use a heredoc to embed snippets of SQL, HTML, or XML code, etc.
You can use a heredoc to embed snippets of SQL, HTML, or XML code, etc.
**Example**

5
docs/ru/development/build-osx.md
View File

@ -2,8 +2,13 @@
toc_priority: 65
toc_title: Сборка на Mac OS X
---
# Как собрать ClickHouse на Mac OS X {#how-to-build-clickhouse-on-mac-os-x}
!!! info "Вам не нужно собирать ClickHouse самостоятельно"
Вы можете установить предварительно собранный ClickHouse, как описано в [Быстром старте](https://clickhouse.com/#quick-start).
Следуйте инструкциям по установке для `macOS (Intel)` или `macOS (Apple Silicon)`.
Сборка должна запускаться с x86_64 (Intel) на macOS версии 10.15 (Catalina) и выше в последней версии компилятора Xcode's native AppleClang, Homebrew's vanilla Clang или в GCC-компиляторах.
## Установка Homebrew {#install-homebrew}

6
docs/ru/sql-reference/syntax.md
View File

@ -28,7 +28,7 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def')
## Комментарии {#comments}
Поддерживаются комментарии в SQL-стиле и C-стиле.
Комментарии в SQL-стиле: от `--` до конца строки. Пробел после `--` может не ставиться.
Комментарии в SQL-стиле: от `--`, `#!` или `# ` до конца строки. Пробел после `--` и `#!` может не ставиться.
Комментарии в C-стиле: от `/*` до `*/`. Такие комментарии могут быть многострочными. Пробелы тоже не обязательны.
## Ключевые слова {#syntax-keywords}
@ -104,9 +104,9 @@ INSERT INTO t VALUES (1, 'Hello, world'), (2, 'abc'), (3, 'def')
### Heredoc {#heredeoc}
Синтаксис [heredoc](https://ru.wikipedia.org/wiki/Heredoc-синтаксис) — это способ определения строк с сохранением исходного формата (часто с переносом строки). `Heredoc` задается как произвольный строковый литерал между двумя символами `$`, например `$heredoc$`. Значение между двумя `heredoc` обрабатывается "как есть".
Синтаксис [heredoc](https://ru.wikipedia.org/wiki/Heredoc-синтаксис) — это способ определения строк с сохранением исходного формата (часто с переносом строки). `Heredoc` задается как произвольный строковый литерал между двумя символами `$`, например `$heredoc$`. Значение между двумя `heredoc` обрабатывается "как есть".
Синтаксис `heredoc` часто используют для вставки кусков кода SQL, HTML, XML и т.п.
Синтаксис `heredoc` часто используют для вставки кусков кода SQL, HTML, XML и т.п.
**Пример**

5
docs/tools/single_page.py
View File

@ -90,7 +90,10 @@ def concatenate(lang, docs_path, single_page_file, nav):
line)
# If failed to replace the relative link, print to log
if '../' in line:
# But with some exceptions:
# - "../src/" -- for cmake-in-clickhouse.md (link to sources)
# - "../usr/share" -- changelog entry that has "../usr/share/zoneinfo"
if '../' in line and (not '../usr/share' in line) and (not '../src/' in line):
logging.info('Failed to resolve relative link:')
logging.info(path)
logging.info(line)

1
docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md
View File

@ -1 +0,0 @@
../../../en/faq/general/how-do-i-contribute-code-to-clickhouse.md

17
docs/zh/faq/general/how-do-i-contribute-code-to-clickhouse.md Normal file
View File

@ -0,0 +1,17 @@
---
title: 我如何为ClickHouse贡献代码?
toc_hidden: true
toc_priority: 120
---
# 我如何为ClickHouse贡献代码? {#how-do-i-contribute-code-to-clickhouse}
ClickHouse是一个开源项目[在GitHub上开发](https://github.com/ClickHouse/ClickHouse)。
按照惯例，贡献指南发布在源代码库根目录的 [CONTRIBUTING.md](https://github.com/ClickHouse/ClickHouse/blob/master/CONTRIBUTING.md)文件中。
如果你想对ClickHouse提出实质性的改变建议，可以考虑[在GitHub上发布一个问题](https://github.com/ClickHouse/ClickHouse/issues/new/choose)，解释一下你想做什么，先与维护人员和社区讨论一下。[此类RFC问题的例子](https://github.com/ClickHouse/ClickHouse/issues?q=is%3Aissue+is%3Aopen+rfc)。
如果您的贡献与安全相关，也请查看[我们的安全政策](https://github.com/ClickHouse/ClickHouse/security/policy/)。

1
docs/zh/faq/index.md
View File

@ -26,6 +26,7 @@ toc_priority: 76
- **[运维操作](../faq/operations/index.md)**
- [如果想在生产环境部署，需要用哪个版本的 ClickHouse 呢?](../faq/operations/production.md)
- [是否可能从 ClickHouse 数据表中删除所有旧的数据记录?](../faq/operations/delete-old-data.md)
- [ClickHouse支持多区域复制吗?](../faq/operations/multi-region-replication.md)
- **[集成开发](../faq/integration/index.md)**
- [如何从 ClickHouse 导出数据到一个文件?](../faq/integration/file-export.md)
- [如果我用ODBC链接Oracle数据库出现编码问题该怎么办?](../faq/integration/oracle-odbc.md)

1
docs/zh/faq/integration/index.md
View File

@ -1 +0,0 @@
../../../en/faq/integration/index.md

21
docs/zh/faq/integration/index.md Normal file
View File

@ -0,0 +1,21 @@
---
title: 关于集成ClickHouse和其他系统的问题
toc_hidden_folder: true
toc_priority: 4
toc_title: Integration
---
# 关于集成ClickHouse和其他系统的问题 {#question-about-integrating-clickhouse-and-other-systems}
问题:
- [如何从 ClickHouse 导出数据到一个文件?](../../faq/integration/file-export.md)
- [如何导入JSON到ClickHouse?](../../faq/integration/json-import.md)
- [如果我用ODBC链接Oracle数据库出现编码问题该怎么办?](../../faq/integration/oracle-odbc.md)
!!! info "没看到你要找的东西吗?"
查看[其他faq类别](../../faq/index.md)或浏览左边栏中的主要文档文章。
{## [原文](https://clickhouse.com/docs/en/faq/integration/) ##}

1
docs/zh/faq/operations/index.md
View File

@ -1 +0,0 @@
../../../en/faq/operations/index.md

20
docs/zh/faq/operations/index.md Normal file
View File

@ -0,0 +1,20 @@
---
title: 关于操作ClickHouse服务器和集群的问题
toc_hidden_folder: true
toc_priority: 3
toc_title: Operations
---
# 关于操作ClickHouse服务器和集群的问题 {#question-about-operating-clickhouse-servers-and-clusters}
问题:
- [如果想在生产环境部署，需要用哪个版本的 ClickHouse 呢?](../../faq/operations/production.md)
- [是否可能从 ClickHouse 数据表中删除所有旧的数据记录?](../../faq/operations/delete-old-data.md)
- [ClickHouse支持多区域复制吗?](../../faq/operations/multi-region-replication.md)
!!! info "没看到你要找的东西吗?"
查看[其他faq类别](../../faq/index.md)或浏览左边栏中的主要文档文章。
{## [原文](https://clickhouse.com/docs/en/faq/production/) ##}

1
docs/zh/faq/operations/multi-region-replication.md
View File

@ -1 +0,0 @@
../../../en/faq/operations/multi-region-replication.md

14
docs/zh/faq/operations/multi-region-replication.md Normal file
View File

@ -0,0 +1,14 @@
---
title: ClickHouse支持多区域复制吗?
toc_hidden: true
toc_priority: 30
---
# ClickHouse支持多区域复制吗? {#does-clickhouse-support-multi-region-replication}
简短的回答是“是的”。然而，我们建议将所有区域/数据中心之间的延迟保持在两位数字范围内，否则，在通过分布式共识协议时，写性能将受到影响。例如，美国海岸之间的复制可能会很好，但美国和欧洲之间就不行。
在配置方面，这与单区域复制没有区别，只是使用位于不同位置的主机作为副本。
更多信息，请参见[关于数据复制的完整文章](../../engines/table-engines/mergetree-family/replication.md)。

343
docs/zh/sql-reference/ansi.md
View File

@ -1,180 +1,189 @@
---
machine_translated: true
machine_translated_rev: ad252bbb4f7e2899c448eb42ecc39ff195c8faa1
toc_priority: 40
toc_title: "ANSI\u517C\u5BB9\u6027"
---
# Ansi Sql兼容性的ClickHouse SQL方言 {#ansi-sql-compatibility-of-clickhouse-sql-dialect}
# ClickHouse SQL方言 与ANSI SQL的兼容性{#ansi-sql-compatibility-of-clickhouse-sql-dialect}
!!! note "注"
本文依赖于表38, “Feature taxonomy and definition for mandatory features”, Annex F of ISO/IEC CD 9075-2:2013.
本文参考Annex G所著的[ISO/IEC CD 9075-2:2011](https://www.iso.org/obp/ui/#iso:std:iso-iec:9075:-2:ed-4:v1:en:sec:8)标准.
## 行为差异 {#differences-in-behaviour}
下表列出了查询功能在ClickHouse中有效但不符合ANSI SQL标准的情况。
下表列出了ClickHouse能够使用，但与ANSI SQL规定有差异的查询特性。
| Feature ID | 功能名称 | 差异 |
|------------|--------------------|---------------------------------------------------------------------|
| E011 | 数值（Numeric）数据类型 | 带小数点的数值文字被解释为近似值 (`Float64`）而不是精确值 (`Decimal`) |
| E051-05 | SELECT字段可以重命名 | 字段不仅仅在SELECT结果中可被重命名 |
| E141-01 | 非空约束 | 表中每一列默认为`NOT NULL` |
| E011-04 | 算术运算符 | ClickHouse不会检查算法，并根据自定义规则更改结果数据类型，而是会溢出 |
| 功能ID | 功能名称 | 差异 |
| ------- | --------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
| E011 | 数值型数据类型 | 带小数点的数字被视为近似值 (`Float64`）而不是精确值 (`Decimal`) |
| E051-05 | SELECT 的列可以重命名 | 字段重命名的作用范围不限于进行重命名的SELECT子查询（参考[表达式别名](https://clickhouse.com/docs/zh/sql-reference/syntax/#notes-on-usage)） |
| E141-01 | NOT NULL（非空）约束 | ClickHouse表中每一列默认为`NOT NULL` |
| E011-04 | 算术运算符 | ClickHouse在运算时会进行溢出，而不是四舍五入。此外会根据自定义规则修改结果数据类型（参考[溢出检查](https://clickhouse.com/docs/zh/sql-reference/data-types/decimal/#yi-chu-jian-cha)） |
## 功能匹配 {#feature-status}
## 功能状态 {#feature-status}
| Feature ID | 功能名称 | 匹配 | 评论 |
|------------|----------------------------------------------------------------|--------------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
| **E011** | **数字数据类型** | **部分**{.text-warning} | |
| E011-01 | 整型和小型数据类型 | 是 {.text-success} | |
| E011-02 | 真实、双精度和浮点数据类型数据类型 | 部分 {.text-warning} | `FLOAT(<binary_precision>)`, `REAL` 和 `DOUBLE PRECISION` 不支持 |
| E011-03 | 十进制和数值数据类型 | 部分 {.text-warning} | 只有 `DECIMAL(p,s)` 支持，而不是 `NUMERIC` |
| E011-04 | 算术运算符 | 是 {.text-success} | |
| E011-05 | 数字比较 | 是 {.text-success} | |
| E011-06 | 数字数据类型之间的隐式转换 | 否。 {.text-danger} | ANSI SQL允许在数值类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 |
| **E021** | **字符串类型** | **部分**{.text-warning} | |
| E021-01 | 字符数据类型 | 否。 {.text-danger} | |
| E021-02 | 字符变化数据类型 | 否。 {.text-danger} | `String` 行为类似，但括号中没有长度限制 |
| E021-03 | 字符文字 | 部分 {.text-warning} | 不自动连接连续文字和字符集支持 |
| E021-04 | 字符长度函数 | 部分 {.text-warning} | 非也。 `USING` 条款 |
| E021-05 | OCTET_LENGTH函数 | 非也。 {.text-danger} | `LENGTH` 表现类似 |
| E021-06 | SUBSTRING | 部分 {.text-warning} | 不支持 `SIMILAR` 和 `ESCAPE` 条款，否 `SUBSTRING_REGEX` 备选案文 |
| E021-07 | 字符串联 | 部分 {.text-warning} | 非也。 `COLLATE` 条款 |
| E021-08 | 上下功能 | 是 {.text-success} | |
| E021-09 | 修剪功能 | 是 {.text-success} | |
| E021-10 | 固定长度和可变长度字符串类型之间的隐式转换 | 否。 {.text-danger} | ANSI SQL允许在字符串类型之间进行任意隐式转换，而ClickHouse依赖于具有多个重载的函数而不是隐式转换 |
| E021-11 | 职位功能 | 部分 {.text-warning} | 不支持 `IN` 和 `USING` 条款，否 `POSITION_REGEX` 备选案文 |
| E021-12 | 字符比较 | 是 {.text-success} | |
| **E031** | **标识符** | **部分**{.text-warning} | |
| E031-01 | 分隔标识符 | 部分 {.text-warning} | Unicode文字支持有限 |
| E031-02 | 小写标识符 | 是 {.text-success} | |
| E031-03 | 尾部下划线 | 是 {.text-success} | |
| **E051** | **基本查询规范** | **部分**{.text-warning} | |
| E051-01 | SELECT DISTINCT | 是 {.text-success} | |
| E051-02 | GROUP BY子句 | 是 {.text-success} | |
| E051-04 | 分组依据可以包含不在列 `<select list>` | 是 {.text-success} | |
| E051-05 | 选择项目可以重命名 | 是 {.text-success} | |
| E051-06 | 有条款 | 是 {.text-success} | |
| E051-07 | 合格\*在选择列表中 | 是 {.text-success} | |
| E051-08 | FROM子句中的关联名称 | 是 {.text-success} | |
| E051-09 | 重命名FROM子句中的列 | 否。 {.text-danger} | |
| **E061** | **基本谓词和搜索条件** | **部分**{.text-warning} | |
| E061-01 | 比较谓词 | 是 {.text-success} | |
| E061-02 | 谓词之间 | 部分 {.text-warning} | 非也。 `SYMMETRIC` 和 `ASYMMETRIC` 条款 |
| E061-03 | 在具有值列表的谓词中 | 是 {.text-success} | |
| E061-04 | 像谓词 | 是 {.text-success} | |
| E061-05 | LIKE谓词：逃避条款 | 否。 {.text-danger} | |
| E061-06 | 空谓词 | 是 {.text-success} | |
| E061-07 | 量化比较谓词 | 非也。 {.text-danger} | |
| E061-08 | 存在谓词 | 非也。 {.text-danger} | |
| E061-09 | 比较谓词中的子查询 | 是 {.text-success} | |
| E061-11 | 谓词中的子查询 | 是 {.text-success} | |
| E061-12 | 量化比较谓词中的子查询 | 否。 {.text-danger} | |
| E061-13 | 相关子查询 | 否。 {.text-danger} | |
| E061-14 | 搜索条件 | 是 {.text-success} | |
| **E071** | **基本查询表达式** | **部分**{.text-warning} | |
| E071-01 | UNION DISTINCT table运算符 | 否。 {.text-danger} | |
| E071-02 | 联合所有表运算符 | 是 {.text-success} | |
| E071-03 | 除了不同的表运算符 | 非也。 {.text-danger} | |
| E071-05 | 通过表运算符组合的列不必具有完全相同的数据类型 | 是 {.text-success} | |
| E071-06 | 子查询中的表运算符 | 是 {.text-success} | |
| **E081** | **基本特权** | **部分**{.text-warning} | 正在进行的工作 |
| **E091** | **设置函数** | **是**{.text-success} | |
| E091-01 | AVG | 是 {.text-success} | |
| E091-02 | COUNT | 是 {.text-success} | |
| E091-03 | MAX | 是 {.text-success} | |
| E091-04 | MIN | 是 {.text-success} | |
| E091-05 | SUM | 是 {.text-success} | |
| E091-06 | 全部量词 | 否。 {.text-danger} | |
| E091-07 | 不同的量词 | 部分 {.text-warning} | 并非所有聚合函数都受支持 |
| **E101** | **基本数据操作** | **部分**{.text-warning} | |
| E101-01 | 插入语句 | 是 {.text-success} | 注：ClickHouse中的主键并不意味着 `UNIQUE` 约束 |
| E101-03 | 搜索更新语句 | 否。 {.text-danger} | 有一个 `ALTER UPDATE` 批量数据修改语句 |
| E101-04 | 搜索的删除语句 | 否。 {.text-danger} | 有一个 `ALTER DELETE` 批量数据删除声明 |
| **E111** | **单行SELECT语句** | **否。**{.text-danger} | |
| **E121** | **基本光标支持** | **否。**{.text-danger} | |
| E121-01 | DECLARE CURSOR | 否。 {.text-danger} | |
| E121-02 | 按列排序不需要在选择列表中 | 否。 {.text-danger} | |
| E121-03 | 按顺序排列的值表达式 | 否。 {.text-danger} | |
| E121-04 | 公开声明 | 否。 {.text-danger} | |
| E121-06 | 定位更新语句 | 否。 {.text-danger} | |
| E121-07 | 定位删除语句 | 否。 {.text-danger} | |
| E121-08 | 关闭声明 | 否。 {.text-danger} | |
| E121-10 | FETCH语句：隐式NEXT | 否。 {.text-danger} | |
| E121-17 | 使用保持游标 | 否。 {.text-danger} | |
| **E131** | **空值支持（空值代替值)** | **部分**{.text-warning} | 一些限制适用 |
| **E141** | **基本完整性约束** | **部分**{.text-warning} | |
| E141-01 | 非空约束 | 是 {.text-success} | 注: `NOT NULL` 默认情况下，表列隐含 |
| E141-02 | 非空列的唯一约束 | 否。 {.text-danger} | |
| E141-03 | 主键约束 | 否。 {.text-danger} | |
| E141-04 | 对于引用删除操作和引用更新操作，具有默认无操作的基本外键约束 | 否。 {.text-danger} | |
| E141-06 | 检查约束 | 是 {.text-success} | |
| E141-07 | 列默认值 | 是 {.text-success} | |
| E141-08 | 在主键上推断为非NULL | 是 {.text-success} | |
| E141-10 | 可以按任何顺序指定外键中的名称 | 否。 {.text-danger} | |
| **E151** | **交易支持** | **否。**{.text-danger} | |
| E151-01 | 提交语句 | 否。 {.text-danger} | |
| E151-02 | 回滚语句 | 否。 {.text-danger} | |
| **E152** | **基本设置事务语句** | **否。**{.text-danger} | |
| E152-01 | SET TRANSACTION语句：隔离级别SERIALIZABLE子句 | 否。 {.text-danger} | |
| E152-02 | SET TRANSACTION语句：只读和读写子句 | 否。 {.text-danger} | |
| **E153** | **具有子查询的可更新查询** | **否。**{.text-danger} | |
| **E161** | **SQL注释使用前导双减** | **是**{.text-success} | |
| **E171** | **SQLSTATE支持** | **否。**{.text-danger} | |
| **E182** | **主机语言绑定** | **否。**{.text-danger} | |
| **F031** | **基本架构操作** | **部分**{.text-warning} | |
| F031-01 | CREATE TABLE语句创建持久基表 | 部分 {.text-warning} | 否。 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 子句，不支持用户解析的数据类型 |
| F031-02 | 创建视图语句 | 部分 {.text-warning} | 否。 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 子句，不支持用户解析的数据类型 |
| F031-03 | 赠款声明 | 是 {.text-success} | |
| F031-04 | ALTER TABLE语句：ADD COLUMN子句 | 部分 {.text-warning} | 不支持 `GENERATED` 条款和系统时间段 |
| F031-13 | DROP TABLE语句：RESTRICT子句 | 否。 {.text-danger} | |
| F031-16 | DROP VIEW语句：RESTRICT子句 | 否。 {.text-danger} | |
| F031-19 | REVOKE语句：RESTRICT子句 | 否。 {.text-danger} | |
| **F041** | **基本连接表** | **部分**{.text-warning} | |
| F041-01 | Inner join（但不一定是INNER关键字) | 是 {.text-success} | |
| F041-02 | 内部关键字 | 是 {.text-success} | |
| F041-03 | LEFT OUTER JOIN | 是 {.text-success} | |
| F041-04 | RIGHT OUTER JOIN | 是 {.text-success} | |
| F041-05 | 可以嵌套外部连接 | 是 {.text-success} | |
| F041-07 | 左侧或右侧外部联接中的内部表也可用于内部联接 | 是 {.text-success} | |
| F041-08 | 支持所有比较运算符（而不仅仅是=) | 否。 {.text-danger} | |
| **F051** | **基本日期和时间** | **部分**{.text-warning} | |
| F051-01 | 日期数据类型（包括对日期文字的支持) | 部分 {.text-warning} | 没有文字 |
| F051-02 | 时间数据类型（包括对时间文字的支持），秒小数精度至少为0 | 否。 {.text-danger} | |
| F051-03 | 时间戳数据类型（包括对时间戳文字的支持），小数秒精度至少为0和6 | 否。 {.text-danger} | `DateTime64` 时间提供了类似的功能 |
| F051-04 | 日期、时间和时间戳数据类型的比较谓词 | 部分 {.text-warning} | 只有一种数据类型可用 |
| F051-05 | Datetime类型和字符串类型之间的显式转换 | 是 {.text-success} | |
| F051-06 | CURRENT_DATE | 否。 {.text-danger} | `today()` 是相似的 |
| F051-07 | LOCALTIME | 否。 {.text-danger} | `now()` 是相似的 |
| F051-08 | LOCALTIMESTAMP | 否。 {.text-danger} | |
| **F081** | **联盟和视图除外** | **部分**{.text-warning} | |
| **F131** | **分组操作** | **部分**{.text-warning} | |
| F131-01 | WHERE、GROUP BY和HAVING子句在具有分组视图的查询中受支持 | 是 {.text-success} | |
| F131-02 | 具有分组视图的查询中支持的多个表 | 是 {.text-success} | |
| F131-03 | 设置具有分组视图的查询中支持的函数 | 是 {.text-success} | |
| F131-04 | 具有分组依据和具有子句和分组视图的子查询 | 是 {.text-success} | |
| F131-05 | 单行选择具有GROUP BY和具有子句和分组视图 | 非也。 {.text-danger} | |
| **F181** | **多模块支持** | **否。**{.text-danger} | |
| **F201** | **投函数** | **是**{.text-success} | |
| **F221** | **显式默认值** | **否。**{.text-danger} | |
| **F261** | **案例表达式** | **是**{.text-success} | |
| F261-01 | 简单案例 | 是 {.text-success} | |
| F261-02 | 检索案例 | 是 {.text-success} | |
| F261-03 | NULLIF | 是 {.text-success} | |
| F261-04 | COALESCE | 是 {.text-success} | |
| **F311** | **架构定义语句** | **部分**{.text-warning} | |
| F311-01 | CREATE SCHEMA | 否。 {.text-danger} | |
| F311-02 | 为持久基表创建表 | 是 {.text-success} | |
| F311-03 | CREATE VIEW | 是 {.text-success} | |
| F311-04 | CREATE VIEW: WITH CHECK OPTION | 否。 {.text-danger} | |
| F311-05 | 赠款声明 | 是 {.text-success} | |
| **F471** | **标量子查询值** | **是**{.text-success} | |
| **F481** | **扩展空谓词** | **是**{.text-success} | |
| **F812** | **基本标记** | **否。**{.text-danger} | |
| **T321** | **基本的SQL调用例程** | **否。**{.text-danger} | |
| T321-01 | 无重载的用户定义函数 | 否。 {.text-danger} | |
| T321-02 | 无重载的用户定义存储过程 | 否。 {.text-danger} | |
| T321-03 | 函数调用 | 否。 {.text-danger} | |
| T321-04 | 电话声明 | 否。 {.text-danger} | |
| T321-05 | 退货声明 | 否。 {.text-danger} | |
| **T631** | **在一个列表元素的谓词中** | **是**{.text-success} | |
| 功能ID | 功能名称 | 状态 | 注释 |
| -------- | ---------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
| **E011** | **数值型数据类型** | **部分**{.text-warning} | |
| E011-01 | INTEGER （整型）和SMALLINT （小整型）数据类型 | 是 {.text-success} | |
| E011-02 | REAL （实数）、DOUBLE PRECISION （双精度浮点数）和FLOAT（单精度浮点数）数据类型数据类型 | 是 {.text-success} | |
| E011-03 | DECIMAL （精确数字）和NUMERIC （精确数字）数据类型 | 是 {.text-success} | |
| E011-04 | 算术运算符 | 是 {.text-success} | |
| E011-05 | 数值比较 | 是 {.text-success} | |
| E011-06 | 数值数据类型之间的隐式转换 | 否 {.text-danger} | ANSI SQL允许在数值类型之间进行任意隐式转换，而ClickHouse针对不同数据类型有对应的比较函数和类型转换函数 |
| **E021** | **字符串类型** | **部分**{.text-warning} | |
| E021-01 | CHARACTER （字符串）数据类型 | 是 {.text-success} | |
| E021-02 | CHARACTER VARYING （可变字符串）数据类型 | 是 {.text-success} | |
| E021-03 | 字符字面量 | 是 {.text-success} | |
| E021-04 | CHARACTER_LENGTH 函数 | 部分 {.text-warning} | 不支持 `using` 从句 |
| E021-05 | OCTET_LENGTH 函数 | 否 {.text-danger} | 使用 `LENGTH` 函数代替 |
| E021-06 | SUBSTRING | 部分 {.text-warning} | 不支持 `SIMILAR` 和 `ESCAPE` 从句，没有`SUBSTRING_REGEX` 函数 |
| E021-07 | 字符串拼接 | 部分 {.text-warning} | 不支持 `COLLATE` 从句 |
| E021-08 | 大小写转换 | 是 {.text-success} | |
| E021-09 | 裁剪字符串 | 是 {.text-success} | |
| E021-10 | 固定长度和可变长度字符串类型之间的隐式转换 | 部分 {.text-warning} | ANSI SQL允许在数据类型之间进行任意隐式转换，而ClickHouse针对不同数据类型有对应的比较函数和类型转换函数 |
| E021-11 | POSITION 函数 | 部分 {.text-warning} | 不支持 `IN` 和 `USING` 从句，不支持`POSITION_REGEX`函数 |
| E021-12 | 字符串比较 | 是 {.text-success} | |
| **E031** | **标识符** | **部分**{.text-warning} | |
| E031-01 | 分隔标识符 | 部分 {.text-warning} | Unicode文字支持有限 |
| E031-02 | 小写标识符 | 是 {.text-success} | |
| E031-03 | 标识符最后加下划线 | 是 {.text-success} | |
| **E051** | **基本查询规范** | **部分**{.text-warning} | |
| E051-01 | SELECT DISTINCT | 是 {.text-success} | |
| E051-02 | GROUP BY 从句 | 是 {.text-success} | |
| E051-04 | GROUP BY 从句中的列可以包含不在 `<select list>`中出现的列 | 是 {.text-success} | |
| E051-05 | SELECT 的列可以重命名 | 是 {.text-success} | |
| E051-06 | HAVING 从句 | 是 {.text-success} | |
| E051-07 | SELECT 选择的列中允许出现\* | 是 {.text-success} | |
| E051-08 | FROM 从句中的关联名称 | 是 {.text-success} | |
| E051-09 | 重命名 FROM 从句中的列 | 否 {.text-danger} | |
| **E061** | **基本谓词和搜索条件** | **部分**{.text-warning} | |
| E061-01 | 比较谓词 | 是 {.text-success} | |
| E061-02 | BETWEEN 谓词 | 部分 {.text-warning} | 不支持 `SYMMETRIC` 和 `ASYMMETRIC` 从句 |
| E061-03 | IN 谓词后可接值列表 | 是 {.text-success} | |
| E061-04 | LIKE 谓词 | 是 {.text-success} | |
| E061-05 | LIKE 谓词后接 ESCAPE 从句 | 否 {.text-danger} | |
| E061-06 | NULL 谓词 | 是 {.text-success} | |
| E061-07 | 量化比较谓词（ALL、SOME、ANY） | 否 {.text-danger} | |
| E061-08 | EXISTS 谓词 | 否 {.text-danger} | |
| E061-09 | 比较谓词中的子查询 | 是 {.text-success} | |
| E061-11 | IN 谓词中的子查询 | 是 {.text-success} | |
| E061-12 | 量化比较谓词（BETWEEN、IN、LIKE）中的子查询 | 否 {.text-danger} | |
| E061-13 | 相关子查询 | 否 {.text-danger} | |
| E061-14 | 搜索条件 | 是 {.text-success} | |
| **E071** | **基本查询表达式** | **部分**{.text-warning} | |
| E071-01 | UNION DISTINCT 表运算符 | 是 {.text-success} | |
| E071-02 | UNION ALL 表运算符 | 是 {.text-success} | |
| E071-03 | EXCEPT DISTINCT 表运算符 | 否 {.text-danger} | |
| E071-05 | 通过表运算符组合的列不必具有完全相同的数据类型 | 是 {.text-success} | |
| E071-06 | 子查询中的表运算符 | 是 {.text-success} | |
| **E081** | **基本权限** | **是**{.text-success} | |
| E081-01 | 表级别的SELECT（查询）权限 | 是 {.text-success} | |
| E081-02 | DELETE（删除）权限 | 是 {.text-success} | |
| E081-03 | 表级别的INSERT（插入）权限 | 是 {.text-success} | |
| E081-04 | 表级别的UPDATE（更新）权限 | 是 {.text-success} | |
| E081-05 | 列级别的UPDATE（更新）权限 | 是 {.text-success} | |
| E081-06 | 表级别的REFERENCES（引用）权限 | 是 {.text-success} | |
| E081-07 | 列级别的REFERENCES（引用）权限 | 是 {.text-success} | |
| E081-08 | WITH GRANT OPTION | 是 {.text-success} | |
| E081-09 | USAGE（使用）权限 | 是 {.text-success} | |
| E081-10 | EXECUTE（执行）权限 | 是 {.text-success} | |
| **E091** | **集合函数** | **是**{.text-success} | |
| E091-01 | AVG | 是 {.text-success} | |
| E091-02 | COUNT | 是 {.text-success} | |
| E091-03 | MAX | 是 {.text-success} | |
| E091-04 | MIN | 是 {.text-success} | |
| E091-05 | SUM | 是 {.text-success} | |
| E091-06 | ALL修饰词 | 否。 {.text-danger} | |
| E091-07 | DISTINCT修饰词 | 是 {.text-success} | 并非所有聚合函数都支持该修饰词 |
| **E101** | **基本数据操作** | **部分**{.text-warning} | |
| E101-01 | INSERT（插入）语句 | 是 {.text-success} | 注：ClickHouse中的主键并不隐含`UNIQUE` 约束 |
| E101-03 | 可指定范围的UPDATE（更新）语句 | 部分 {.text-warning} | `ALTER UPDATE` 语句用来批量更新数据 |
| E101-04 | 可指定范围的DELETE（删除）语句 | 部分 {.text-warning} | `ALTER DELETE` 语句用来批量删除数据 |
| **E111** | **返回一行的SELECT语句** | **否**{.text-danger} | |
| **E121** | **基本游标支持** | **否**{.text-danger} | |
| E121-01 | DECLARE CURSOR | 否 {.text-danger} | |
| E121-02 | ORDER BY 涉及的列不需要出现在SELECT的列中 | 是 {.text-success} | |
| E121-03 | ORDER BY 从句中的表达式 | 是 {.text-success} | |
| E121-04 | OPEN 语句 | 否 {.text-danger} | |
| E121-06 | 受游标位置控制的 UPDATE 语句 | 否 {.text-danger} | |
| E121-07 | 受游标位置控制的 DELETE 语句 | 否 {.text-danger} | |
| E121-08 | CLOSE 语句 | 否 {.text-danger} | |
| E121-10 | FETCH 语句中包含隐式NEXT | 否 {.text-danger} | |
| E121-17 | WITH HOLD 游标 | 否 {.text-danger} | |
| **E131** | **空值支持** | **是**{.text-success} | 有部分限制 |
| **E141** | **基本完整性约束** | **部分**{.text-warning} | |
| E141-01 | NOT NULL（非空）约束 | 是 {.text-success} | 注: 默认情况下ClickHouse表中的列隐含`NOT NULL`约束 |
| E141-02 | NOT NULL（非空）列的UNIQUE（唯一）约束 | 否 {.text-danger} | |
| E141-03 | 主键约束 | 部分 {.text-warning} | |
| E141-04 | 对于引用删除和引用更新操作，基本的FOREIGN KEY（外键）约束默认不进行任何操作（NO ACTION） | 否 {.text-danger} | |
| E141-06 | CHECK（检查）约束 | 是 {.text-success} | |
| E141-07 | 列默认值 | 是 {.text-success} | |
| E141-08 | 在主键上推断非空 | 是 {.text-success} | |
| E141-10 | 可以按任何顺序指定外键中的名称 | 否 {.text-danger} | |
| **E151** | **事务支持** | **否**{.text-danger} | |
| E151-01 | COMMIT（提交）语句 | 否 {.text-danger} | |
| E151-02 | ROLLBACK（回滚）语句 | 否 {.text-danger} | |
| **E152** | **基本的SET TRANSACTION（设置事务隔离级别）语句** | **否**{.text-danger} | |
| E152-01 | SET TRANSACTION语句：ISOLATION LEVEL SERIALIZABLE（隔离级别为串行化）从句 | 否 {.text-danger} | |
| E152-02 | SET TRANSACTION语句：READ ONLY（只读）和READ WRITE（读写）从句 | 否 {.text-danger} | |
| **E153** | **具有子查询的可更新查询** | **是**{.text-success} | |
| **E161** | **使用“--”符号作为SQL注释** | **是**{.text-success} | |
| **E171** | **SQLSTATE支持** | **否**{.text-danger} | |
| **E182** | **主机语言绑定** | **否**{.text-danger} | |
| **F031** | **基本架构操作** | **部分**{.text-warning} | |
| F031-01 | 使用 CREATE TABLE 语句创建持久表 | 部分 {.text-warning} | 不支持 `SYSTEM VERSIONING`, `ON COMMIT`, `GLOBAL`, `LOCAL`, `PRESERVE`, `DELETE`, `REF IS`, `WITH OPTIONS`, `UNDER`, `LIKE`, `PERIOD FOR` 从句，不支持用户解析的数据类型 |
| F031-02 | CREATE VIEW（创建视图）语句 | 部分 {.text-warning} | 不支持 `RECURSIVE`, `CHECK`, `UNDER`, `WITH OPTIONS` 从句，不支持用户解析的数据类型 |
| F031-03 | GRANT（授权）语句 | 是 {.text-success} | |
| F031-04 | ALTER TABLE语句：ADD COLUMN从句 | 是 {.text-success} | 不支持 `GENERATED` 从句和以系统时间做参数 |
| F031-13 | DROP TABLE语句：RESTRICT从句 | 否 {.text-danger} | |
| F031-16 | DROP VIEW语句：RESTRICT子句 | 否 {.text-danger} | |
| F031-19 | REVOKE语句：RESTRICT子句 | 否 {.text-danger} | |
| **F041** | **基本连接关系** | **部分**{.text-warning} | |
| F041-01 | Inner join（但不一定是INNER关键字) | 是 {.text-success} | |
| F041-02 | INNER 关键字 | 是 {.text-success} | |
| F041-03 | LEFT OUTER JOIN | 是 {.text-success} | |
| F041-04 | RIGHT OUTER JOIN | 是 {.text-success} | |
| F041-05 | 外连接可嵌套 | 是 {.text-success} | |
| F041-07 | 左外部连接或右外连接中的内部表也可用于内部联接 | 是 {.text-success} | |
| F041-08 | 支持所有比较运算符（而不仅仅是=) | 否 {.text-danger} | |
| **F051** | **基本日期和时间** | **部分**{.text-warning} | |
| F051-01 | DATE（日期）数据类型（并支持用于表达日期的字面量) | 是 {.text-success} | |
| F051-02 | TIME（时间）数据类型（并支持用于表达时间的字面量），小数秒精度至少为0 | 否 {.text-danger} | |
| F051-03 | 时间戳数据类型（并支持用于表达时间戳的字面量），小数秒精度至少为0和6 | 是 {.text-danger} | |
| F051-04 | 日期、时间和时间戳数据类型的比较谓词 | 是 {.text-success} | |
| F051-05 | Datetime 类型和字符串形式表达的时间之间的显式转换 | 是 {.text-success} | |
| F051-06 | CURRENT_DATE | 否 {.text-danger} | 使用`today()`替代 |
| F051-07 | LOCALTIME | 否 {.text-danger} | 使用`now()`替代 |
| F051-08 | LOCALTIMESTAMP | 否 {.text-danger} | |
| **F081** | **视图的UNION和EXCEPT操作** | **部分**{.text-warning} | |
| **F131** | **分组操作** | **部分**{.text-warning} | |
| F131-01 | 在具有分组视图的查询中支持 WHERE、GROUP BY 和 HAVING 子句 | 是 {.text-success} | |
| F131-02 | 在分组视图中支持多张表 | 是 {.text-success} | |
| F131-03 | 分组视图的查询中支持集合函数 | 是 {.text-success} | |
| F131-04 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的子查询 | 是 {.text-success} | |
| F131-05 | 带有 `GROUP BY` 和 `HAVING` 从句，以及分组视图的仅返回1条记录的SELECT查询 | 否 {.text-danger} | |
| **F181** | **多模块支持** | **否**{.text-danger} | |
| **F201** | **CAST 函数** | **是**{.text-success} | |
| **F221** | **显式默认值** | **否**{.text-danger} | |
| **F261** | **CASE 表达式** | **是**{.text-success} | |
| F261-01 | 简单 CASE 表达式 | 是 {.text-success} | |
| F261-02 | 搜索型 CASE 表达式 | 是 {.text-success} | |
| F261-03 | NULLIF | 是 {.text-success} | |
| F261-04 | COALESCE | 是 {.text-success} | |
| **F311** | **架构定义语句** | **部分**{.text-warning} | |
| F311-01 | CREATE SCHEMA | 部分 {.text-warning} | 见`CREATE DATABASE` |
| F311-02 | 用于创建持久表的 CREATE TABLE | 是 {.text-success} | |
| F311-03 | CREATE VIEW | 是 {.text-success} | |
| F311-04 | CREATE VIEW: WITH CHECK OPTION | 否 {.text-danger} | |
| F311-05 | GRANT 语句 | 是 {.text-success} | |
| **F471** | **标量子查询** | **是**{.text-success} | |
| **F481** | **扩展 NULL 谓词** | **是**{.text-success} | |
| **F812** | **基本标志位** | **否**{.text-danger} |
| **S011** | **用于不重复数据的数据类型** | **否**{.text-danger} |
| **T321** | **基本的SQL调用例程** | **否**{.text-danger} | |
| T321-01 | 没有重载的用户定义函数 | 否{.text-danger} | |
| T321-02 | 没有重载的用户定义存储过程 | 否{.text-danger} | |
| T321-03 | 功能调用 | 否 {.text-danger} | |
| T321-04 | CALL 语句 | 否 {.text-danger} | |
| T321-05 | RETURN 语句 | 否 {.text-danger} | |
| **T631** | **IN 谓词后接一个列表** | **是**{.text-success} | |

2
programs/client/Client.cpp
View File

@ -286,7 +286,7 @@ bool Client::executeMultiQuery(const String & all_queries_text)
// , where the inline data is delimited by semicolon and not by a
// newline.
auto * insert_ast = parsed_query->as<ASTInsertQuery>();
if (insert_ast && insert_ast->data)
if (insert_ast && isSyncInsertWithData(*insert_ast, global_context))
{
this_query_end = insert_ast->end;
adjustQueryEnd(this_query_end, all_queries_end, global_context->getSettingsRef().max_parser_depth);

2
programs/keeper/Keeper.cpp
View File

@ -324,7 +324,7 @@ int Keeper::main(const std::vector<std::string> & /*args*/)
}
else
{
LOG_WARNING(log, message);
LOG_WARNING(log, fmt::runtime(message));
}
}

8
programs/library-bridge/Handlers.cpp
View File

@ -37,7 +37,7 @@ namespace
if (!response.sent())
*response.send() << message << std::endl;
LOG_WARNING(&Poco::Logger::get("LibraryBridge"), message);
LOG_WARNING(&Poco::Logger::get("LibraryBridge"), fmt::runtime(message));
}
std::shared_ptr<Block> parseColumns(std::string && column_string)
@ -123,7 +123,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
}
else
{
LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead");
LOG_TRACE(log, "Cannot clone from dictionary with id: {}, will call libNew instead", from_dictionary_id);
lib_new = true;
}
}
@ -178,7 +178,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
catch (const Exception & ex)
{
processError(response, "Invalid 'sample_block' parameter in request body '" + ex.message() + "'");
LOG_WARNING(log, ex.getStackTraceString());
LOG_WARNING(log, fmt::runtime(ex.getStackTraceString()));
return;
}
@ -278,7 +278,7 @@ void LibraryRequestHandler::handleRequest(HTTPServerRequest & request, HTTPServe
catch (const Exception & ex)
{
processError(response, "Invalid 'requested_block' parameter in request body '" + ex.message() + "'");
LOG_WARNING(log, ex.getStackTraceString());
LOG_WARNING(log, fmt::runtime(ex.getStackTraceString()));
return;
}

2
programs/odbc-bridge/ColumnInfoHandler.cpp
View File

@ -77,7 +77,7 @@ void ODBCColumnsInfoHandler::handleRequest(HTTPServerRequest & request, HTTPServ
response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
if (!response.sent())
*response.send() << message << std::endl;
LOG_WARNING(log, message);
LOG_WARNING(log, fmt::runtime(message));
};
if (!params.has("table"))

2
programs/odbc-bridge/IdentifierQuoteHandler.cpp
View File

@ -29,7 +29,7 @@ void IdentifierQuoteHandler::handleRequest(HTTPServerRequest & request, HTTPServ
response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
if (!response.sent())
*response.send() << message << std::endl;
LOG_WARNING(log, message);
LOG_WARNING(log, fmt::runtime(message));
};
if (!params.has("connection_string"))

4
programs/odbc-bridge/MainHandler.cpp
View File

@ -46,7 +46,7 @@ void ODBCHandler::processError(HTTPServerResponse & response, const std::string
response.setStatusAndReason(HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
if (!response.sent())
*response.send() << message << std::endl;
LOG_WARNING(log, message);
LOG_WARNING(log, fmt::runtime(message));
}
@ -102,7 +102,7 @@ void ODBCHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
catch (const Exception & ex)
{
processError(response, "Invalid 'sample_block' parameter in request body '" + ex.message() + "'");
LOG_ERROR(log, ex.getStackTraceString());
LOG_ERROR(log, fmt::runtime(ex.getStackTraceString()));
return;
}

2
programs/odbc-bridge/SchemaAllowedHandler.cpp
View File

@ -37,7 +37,7 @@ void SchemaAllowedHandler::handleRequest(HTTPServerRequest & request, HTTPServer
response.setStatusAndReason(Poco::Net::HTTPResponse::HTTP_INTERNAL_SERVER_ERROR);
if (!response.sent())
*response.send() << message << std::endl;
LOG_WARNING(log, message);
LOG_WARNING(log, fmt::runtime(message));
};
if (!params.has("connection_string"))

44
src/Access/CachedAccessChecking.cpp Normal file
View File

@ -0,0 +1,44 @@
#include <Access/CachedAccessChecking.h>
#include <Access/ContextAccess.h>
namespace DB
{
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, AccessFlags access_flags_)
: CachedAccessChecking(access_, AccessRightsElement{access_flags_})
{
}
CachedAccessChecking::CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, const AccessRightsElement & element_)
: access(access_), element(element_)
{
}
CachedAccessChecking::~CachedAccessChecking() = default;
bool CachedAccessChecking::checkAccess(bool throw_if_denied)
{
if (checked)
return result;
if (throw_if_denied)
{
try
{
access->checkAccess(element);
result = true;
}
catch (...)
{
result = false;
throw;
}
}
else
{
result = access->isGranted(element);
}
checked = true;
return result;
}
}

29
src/Access/CachedAccessChecking.h Normal file
View File

@ -0,0 +1,29 @@
#pragma once
#include <Access/Common/AccessRightsElement.h>
#include <memory>
namespace DB
{
class ContextAccess;
/// Checks if the current user has a specified access type granted,
/// and if it's checked another time later, it will just return the first result.
class CachedAccessChecking
{
public:
CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, AccessFlags access_flags_);
CachedAccessChecking(const std::shared_ptr<const ContextAccess> & access_, const AccessRightsElement & element_);
~CachedAccessChecking();
bool checkAccess(bool throw_if_denied = true);
private:
const std::shared_ptr<const ContextAccess> access;
const AccessRightsElement element;
bool checked = false;
bool result = false;
};
}

5
src/CMakeLists.txt
View File

@ -475,11 +475,6 @@ if (TARGET ch_contrib::sqlite)
dbms_target_link_libraries(PUBLIC ch_contrib::sqlite)
endif()
if (USE_CASSANDRA)
dbms_target_link_libraries(PUBLIC ${CASSANDRA_LIBRARY})
dbms_target_include_directories (SYSTEM BEFORE PUBLIC ${CASS_INCLUDE_DIR})
endif()
if (TARGET ch_contrib::msgpack)
target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::msgpack)
endif()

38
src/Client/ClientBase.cpp
View File

@ -573,6 +573,18 @@ void ClientBase::updateSuggest(const ASTCreateQuery & ast_create)
suggest->addWords(std::move(new_words));
}
bool ClientBase::isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context)
{
if (!insert_query.data)
return false;
auto settings = context->getSettings();
if (insert_query.settings_ast)
settings.applyChanges(insert_query.settings_ast->as<ASTSetQuery>()->changes);
return !settings.async_insert;
}
void ClientBase::processTextAsSingleQuery(const String & full_query)
{
/// Some parts of a query (result output and formatting) are executed
@ -597,10 +609,12 @@ void ClientBase::processTextAsSingleQuery(const String & full_query)
updateSuggest(*create);
}
// An INSERT query may have the data that follow query text. Remove the
/// Send part of query without data, because data will be sent separately.
auto * insert = parsed_query->as<ASTInsertQuery>();
if (insert && insert->data)
/// An INSERT query may have the data that follows query text.
/// Send part of the query without data, because data will be sent separately.
/// But for asynchronous inserts we don't extract data, because it's needed
/// to be done on server side in that case (for coalescing the data from multiple inserts on server side).
const auto * insert = parsed_query->as<ASTInsertQuery>();
if (insert && isSyncInsertWithData(*insert, global_context))
query_to_execute = full_query.substr(0, insert->data - full_query.data());
else
query_to_execute = full_query;
@ -1261,7 +1275,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
for (const auto & query_id_format : query_id_formats)
{
writeString(query_id_format.first, std_out);
writeString(fmt::format(query_id_format.second, fmt::arg("query_id", global_context->getCurrentQueryId())), std_out);
writeString(fmt::format(fmt::runtime(query_id_format.second), fmt::arg("query_id", global_context->getCurrentQueryId())), std_out);
writeChar('\n', std_out);
std_out.next();
}
@ -1303,8 +1317,10 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
if (insert && insert->select)
insert->tryFindInputFunction(input_function);
bool is_async_insert = global_context->getSettings().async_insert && insert && insert->hasInlinedData();
/// INSERT query for which data transfer is needed (not an INSERT SELECT or input()) is processed separately.
if (insert && (!insert->select || input_function) && !insert->watch)
if (insert && (!insert->select || input_function) && !insert->watch && !is_async_insert)
{
if (input_function && insert->format.empty())
throw Exception("FORMAT must be specified for function input()", ErrorCodes::INVALID_USAGE_OF_INPUT);
@ -1434,17 +1450,17 @@ MultiQueryProcessingStage ClientBase::analyzeMultiQueryText(
// row input formats (e.g. TSV) can't tell when the input stops,
// unlike VALUES.
auto * insert_ast = parsed_query->as<ASTInsertQuery>();
const char * query_to_execute_end = this_query_end;
if (insert_ast && insert_ast->data)
{
this_query_end = find_first_symbols<'\n'>(insert_ast->data, all_queries_end);
insert_ast->end = this_query_end;
query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), insert_ast->data - this_query_begin);
}
else
{
query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), this_query_end - this_query_begin);
query_to_execute_end = isSyncInsertWithData(*insert_ast, global_context) ? insert_ast->data : this_query_end;
}
query_to_execute = all_queries_text.substr(this_query_begin - all_queries_text.data(), query_to_execute_end - this_query_begin);
// Try to include the trailing comment with test hints. It is just
// a guess for now, because we don't yet know where the query ends
// if it is an INSERT query with inline data. We will do it again

2
src/Client/ClientBase.h
View File

@ -139,6 +139,8 @@ private:
void updateSuggest(const ASTCreateQuery & ast_create);
protected:
static bool isSyncInsertWithData(const ASTInsertQuery & insert_query, const ContextPtr & context);
bool is_interactive = false; /// Use either interactive line editing interface or batch mode.
bool is_multiquery = false;
bool delayed_interactive = false;

7
src/Client/ClientBaseHelpers.cpp
View File

@ -6,7 +6,6 @@
#include <Parsers/Lexer.h>
#include <Common/UTF8Helpers.h>
namespace DB
{
@ -114,6 +113,7 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
{TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::VerticalDelimiter, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)},
{TokenType::HereDoc, Replxx::Color::CYAN},
@ -151,6 +151,11 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
for (Token token = lexer.nextToken(); !token.isEnd(); token = lexer.nextToken())
{
if (token.type == TokenType::Semicolon || token.type == TokenType::VerticalDelimiter)
ReplxxLineReader::setLastIsDelimiter(true);
else if (token.type != TokenType::Whitespace)
ReplxxLineReader::setLastIsDelimiter(false);
size_t utf8_len = UTF8::countCodePoints(reinterpret_cast<const UInt8 *>(token.begin), token.size());
for (size_t code_point_index = 0; code_point_index < utf8_len; ++code_point_index)
{

2
src/Client/Connection.cpp
View File

@ -405,7 +405,7 @@ bool Connection::ping()
}
catch (const Poco::Exception & e)
{
LOG_TRACE(log_wrapper.get(), e.displayText());
LOG_TRACE(log_wrapper.get(), fmt::runtime(e.displayText()));
return false;
}

6
src/Client/ConnectionEstablisher.cpp
View File

@ -58,9 +58,9 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::
auto table_status_it = status_response.table_states_by_id.find(*table_to_check);
if (table_status_it == status_response.table_states_by_id.end())
{
const char * message_pattern = "There is no table {}.{} on server: {}";
fail_message = fmt::format(message_pattern, backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription());
LOG_WARNING(log, fail_message);
fail_message = fmt::format("There is no table {}.{} on server: {}",
backQuote(table_to_check->database), backQuote(table_to_check->table), result.entry->getDescription());
LOG_WARNING(log, fmt::runtime(fail_message));
ProfileEvents::increment(ProfileEvents::DistributedConnectionMissingTable);
return;
}

1
src/Common/CurrentMetrics.cpp
View File

@ -80,6 +80,7 @@
M(SyncDrainedConnections, "Number of connections drained synchronously.") \
M(ActiveSyncDrainedConnections, "Number of active connections drained synchronously.") \
M(AsynchronousReadWait, "Number of threads waiting for asynchronous read.") \
M(PendingAsyncInsert, "Number of asynchronous inserts that are waiting for flush.") \
namespace CurrentMetrics
{

2
src/Common/DNSResolver.cpp
View File

@ -272,7 +272,7 @@ bool DNSResolver::updateCacheImpl(UpdateF && update_func, ElemsT && elems, const
}
if (!lost_elems.empty())
LOG_INFO(log, log_msg, lost_elems);
LOG_INFO(log, fmt::runtime(log_msg), lost_elems);
return updated;
}

6
src/Common/Exception.h
View File

@ -37,7 +37,7 @@ public:
// Format message with fmt::format, like the logging functions.
template <typename ...Args>
Exception(int code, const std::string & fmt, Args&&... args)
: Exception(fmt::format(fmt, std::forward<Args>(args)...), code)
: Exception(fmt::format(fmt::runtime(fmt), std::forward<Args>(args)...), code)
{}
struct CreateFromPocoTag {};
@ -55,7 +55,7 @@ public:
template <typename ...Args>
void addMessage(const std::string& format, Args&&... args)
{
extendedMessage(fmt::format(format, std::forward<Args>(args)...));
extendedMessage(fmt::format(fmt::runtime(format), std::forward<Args>(args)...));
}
void addMessage(const std::string& message)
@ -119,7 +119,7 @@ public:
// Format message with fmt::format, like the logging functions.
template <typename ...Args>
ParsingException(int code, const std::string & fmt, Args&&... args)
: Exception(fmt::format(fmt, std::forward<Args>(args)...), code)
: Exception(fmt::format(fmt::runtime(fmt), std::forward<Args>(args)...), code)
{}

1
src/Common/ProfileEvents.cpp
View File

@ -8,6 +8,7 @@
M(Query, "Number of queries to be interpreted and potentially executed. Does not include queries that failed to parse or were rejected due to AST size limits, quota limits or limits on the number of simultaneously running queries. May include internal queries initiated by ClickHouse itself. Does not count subqueries.") \
M(SelectQuery, "Same as Query, but only for SELECT queries.") \
M(InsertQuery, "Same as Query, but only for INSERT queries.") \
M(AsyncInsertQuery, "Same as InsertQuery, but only for asynchronous INSERT queries.") \
M(FailedQuery, "Number of failed queries.") \
M(FailedSelectQuery, "Same as FailedQuery, but only for SELECT queries.") \
M(FailedInsertQuery, "Same as FailedQuery, but only for INSERT queries.") \

2
src/Common/ProgressIndication.cpp
View File

@ -243,7 +243,7 @@ void ProgressIndication::writeProgress()
if (width_of_progress_bar > 0)
{
size_t bar_width = UnicodeBar::getWidth(current_count, 0, max_count, width_of_progress_bar);
double bar_width = UnicodeBar::getWidth(current_count, 0, max_count, width_of_progress_bar);
std::string bar = UnicodeBar::render(bar_width);
/// Render profiling_msg at left on top of the progress bar.

2
src/Common/ZooKeeper/ZooKeeper.cpp
View File

@ -1145,7 +1145,7 @@ std::string normalizeZooKeeperPath(std::string zookeeper_path, bool check_starts
if (check_starts_with_slash)
throw DB::Exception(DB::ErrorCodes::BAD_ARGUMENTS, "ZooKeeper path must starts with '/', got '{}'", zookeeper_path);
if (log)
LOG_WARNING(log, "ZooKeeper path ('{}') does not start with '/'. It will not be supported in future releases");
LOG_WARNING(log, "ZooKeeper path ('{}') does not start with '/'. It will not be supported in future releases", zookeeper_path);
zookeeper_path = "/" + zookeeper_path;
}

2
src/Common/tests/gtest_log.cpp
View File

@ -17,7 +17,7 @@ TEST(Logger, Log)
Poco::Logger * log = &Poco::Logger::get("Log");
/// This test checks that we don't pass this string to fmtlib, because it is the only argument.
EXPECT_NO_THROW(LOG_INFO(log, "Hello {} World"));
EXPECT_NO_THROW(LOG_INFO(log, fmt::runtime("Hello {} World")));
}
TEST(Logger, TestLog)

2
src/Coordination/LoggerWrapper.h
View File

@ -39,7 +39,7 @@ public:
const std::string & msg) override
{
LogsLevel db_level = static_cast<LogsLevel>(level_);
LOG_IMPL(log, db_level, LEVELS.at(db_level), msg);
LOG_IMPL(log, db_level, LEVELS.at(db_level), fmt::runtime(msg));
}
void set_level(int level_) override

1
src/Core/Settings.h
View File

@ -170,6 +170,7 @@ class IColumn;
M(Bool, force_index_by_date, false, "Throw an exception if there is a partition key in a table, and it is not used.", 0) \
M(Bool, force_primary_key, false, "Throw an exception if there is primary key in a table, and it is not used.", 0) \
M(Bool, use_skip_indexes, true, "Use data skipping indexes during query execution.", 0) \
M(Bool, use_skip_indexes_if_final, false, "If query has FINAL, then skipping data based on indexes may produce incorrect result, hence disabled by default.", 0) \
M(String, force_data_skipping_indices, "", "Comma separated list of strings or literals with the name of the data skipping indices that should be used during query execution, otherwise an exception will be thrown.", 0) \
\
M(Float, max_streams_to_max_threads_ratio, 1, "Allows you to use more sources than the number of threads - to more evenly distribute work across threads. It is assumed that this is a temporary solution, since it will be possible in the future to make the number of sources equal to the number of threads, but for each source to dynamically select available work for itself.", 0) \

8
src/Databases/DatabaseAtomic.cpp
View File

@ -80,7 +80,7 @@ void DatabaseAtomic::drop(ContextPtr)
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(true));
LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true)));
}
fs::remove_all(getMetadataPath());
}
@ -469,7 +469,7 @@ void DatabaseAtomic::tryCreateSymlink(const String & table_name, const String &
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(true));
LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true)));
}
}
@ -482,7 +482,7 @@ void DatabaseAtomic::tryRemoveSymlink(const String & table_name)
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(true));
LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true)));
}
}
@ -527,7 +527,7 @@ void DatabaseAtomic::renameDatabase(ContextPtr query_context, const String & new
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(true));
LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(true)));
}
auto new_name_escaped = escapeForFileName(new_name);

2
src/Databases/DatabaseOnDisk.cpp
View File

@ -316,7 +316,7 @@ void DatabaseOnDisk::dropTable(ContextPtr local_context, const String & table_na
}
catch (...)
{
LOG_WARNING(log, getCurrentExceptionMessage(__PRETTY_FUNCTION__));
LOG_WARNING(log, fmt::runtime(getCurrentExceptionMessage(__PRETTY_FUNCTION__)));
attachTable(local_context, table_name, table, table_data_path_relative);
if (renamed)
fs::rename(table_metadata_path_drop, table_metadata_path);

2
src/Databases/SQLite/DatabaseSQLite.cpp
View File

@ -94,7 +94,7 @@ bool DatabaseSQLite::checkSQLiteTable(const String & table_name) const
if (!sqlite_db)
sqlite_db = openSQLiteDB(database_path, getContext(), /* throw_on_error */true);
const String query = fmt::format("SELECT name FROM sqlite_master WHERE type='table' AND name='{table_name}';", table_name);
const String query = fmt::format("SELECT name FROM sqlite_master WHERE type='table' AND name='{}';", table_name);
auto callback_get_data = [](void * res, int, char **, char **) -> int
{

2
src/Databases/SQLite/SQLiteUtils.cpp
View File

@ -20,7 +20,7 @@ void processSQLiteError(const String & message, bool throw_on_error)
if (throw_on_error)
throw Exception(ErrorCodes::PATH_ACCESS_DENIED, message);
else
LOG_ERROR(&Poco::Logger::get("SQLiteEngine"), message);
LOG_ERROR(&Poco::Logger::get("SQLiteEngine"), fmt::runtime(message));
}

10
src/Dictionaries/CassandraHelpers.cpp
View File

@ -58,15 +58,15 @@ void cassandraLogCallback(const CassLogMessage * message, void * data)
{
Poco::Logger * logger = static_cast<Poco::Logger *>(data);
if (message->severity == CASS_LOG_CRITICAL || message->severity == CASS_LOG_ERROR)
LOG_ERROR(logger, message->message);
LOG_ERROR(logger, fmt::runtime(message->message));
else if (message->severity == CASS_LOG_WARN)
LOG_WARNING(logger, message->message);
LOG_WARNING(logger, fmt::runtime(message->message));
else if (message->severity == CASS_LOG_INFO)
LOG_INFO(logger, message->message);
LOG_INFO(logger, fmt::runtime(message->message));
else if (message->severity == CASS_LOG_DEBUG)
LOG_DEBUG(logger, message->message);
LOG_DEBUG(logger, fmt::runtime(message->message));
else if (message->severity == CASS_LOG_TRACE)
LOG_TRACE(logger, message->message);
LOG_TRACE(logger, fmt::runtime(message->message));
}
}

6
src/Dictionaries/MySQLDictionarySource.cpp
View File

@ -193,7 +193,7 @@ Pipe MySQLDictionarySource::loadAll()
auto connection = pool->get();
last_modification = getLastModification(connection, false);
LOG_TRACE(log, load_all_query);
LOG_TRACE(log, fmt::runtime(load_all_query));
return loadFromQuery(load_all_query);
}
@ -203,7 +203,7 @@ Pipe MySQLDictionarySource::loadUpdatedAll()
last_modification = getLastModification(connection, false);
std::string load_update_query = getUpdateFieldAndDate();
LOG_TRACE(log, load_update_query);
LOG_TRACE(log, fmt::runtime(load_update_query));
return loadFromQuery(load_update_query);
}
@ -289,7 +289,7 @@ LocalDateTime MySQLDictionarySource::getLastModification(mysqlxx::Pool::Entry &
{
auto query = connection->query("SHOW TABLE STATUS LIKE " + quoteForLike(configuration.table));
LOG_TRACE(log, query.str());
LOG_TRACE(log, fmt::runtime(query.str()));
auto result = query.use();

4
src/Dictionaries/PostgreSQLDictionarySource.cpp
View File

@ -80,7 +80,7 @@ PostgreSQLDictionarySource::PostgreSQLDictionarySource(const PostgreSQLDictionar
Pipe PostgreSQLDictionarySource::loadAll()
{
LOG_TRACE(log, load_all_query);
LOG_TRACE(log, fmt::runtime(load_all_query));
return loadBase(load_all_query);
}
@ -88,7 +88,7 @@ Pipe PostgreSQLDictionarySource::loadAll()
Pipe PostgreSQLDictionarySource::loadUpdatedAll()
{
auto load_update_query = getUpdateFieldAndDate();
LOG_TRACE(log, load_update_query);
LOG_TRACE(log, fmt::runtime(load_update_query));
return loadBase(load_update_query);
}

4
src/Dictionaries/XDBCDictionarySource.cpp
View File

@ -121,7 +121,7 @@ std::string XDBCDictionarySource::getUpdateFieldAndDate()
Pipe XDBCDictionarySource::loadAll()
{
LOG_TRACE(log, load_all_query);
LOG_TRACE(log, fmt::runtime(load_all_query));
return loadFromQuery(bridge_url, sample_block, load_all_query);
}
@ -130,7 +130,7 @@ Pipe XDBCDictionarySource::loadUpdatedAll()
{
std::string load_query_update = getUpdateFieldAndDate();
LOG_TRACE(log, load_query_update);
LOG_TRACE(log, fmt::runtime(load_query_update));
return loadFromQuery(bridge_url, sample_block, load_query_update);
}

2
src/Disks/DiskSelector.cpp
View File

@ -101,7 +101,7 @@ DiskSelectorPtr DiskSelector::updateFromConfig(
}
writeString(" disappeared from configuration, this change will be applied after restart of ClickHouse", warning);
LOG_WARNING(&Poco::Logger::get("DiskSelector"), warning.str());
LOG_WARNING(&Poco::Logger::get("DiskSelector"), fmt::runtime(warning.str()));
}
return result;

2
src/Functions/logTrace.cpp
View File

@ -48,7 +48,7 @@ namespace
"First argument for function " + getName() + " must be Constant string", ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT);
static auto * log = &Poco::Logger::get("FunctionLogTrace");
LOG_TRACE(log, message);
LOG_TRACE(log, fmt::runtime(message));
return DataTypeUInt8().createColumnConst(input_rows_count, 0);
}

2
src/IO/S3Common.cpp
View File

@ -317,7 +317,7 @@ public:
, load_frequency_ms(Aws::Auth::REFRESH_THRESHOLD)
, logger(&Poco::Logger::get("AWSInstanceProfileCredentialsProvider"))
{
LOG_INFO(logger, "Creating Instance with injected EC2MetadataClient and refresh rate {}.");
LOG_INFO(logger, "Creating Instance with injected EC2MetadataClient and refresh rate.");
}
Aws::Auth::AWSCredentials GetAWSCredentials() override

70
src/IO/parseDateTimeBestEffort.cpp
View File

@ -194,7 +194,7 @@ ReturnType parseDateTimeBestEffortImpl(
}
else if (num_digits == 6)
{
/// This is YYYYMM
/// This is YYYYMM or hhmmss
if (!year && !month)
{
readDecimalNumber<4>(year, digits);
@ -435,47 +435,59 @@ ReturnType parseDateTimeBestEffortImpl(
else if (c == '+' || c == '-')
{
++in.position();
has_time_zone_offset = true;
if (c == '-')
time_zone_offset_negative = true;
num_digits = readDigits(digits, sizeof(digits), in);
if (num_digits == 4)
if (num_digits == 6 && !has_time && year && month && day_of_month)
{
readDecimalNumber<2>(time_zone_offset_hour, digits);
readDecimalNumber<2>(time_zone_offset_minute, digits + 2);
}
else if (num_digits == 3)
{
readDecimalNumber<1>(time_zone_offset_hour, digits);
readDecimalNumber<2>(time_zone_offset_minute, digits + 1);
}
else if (num_digits == 2)
{
readDecimalNumber<2>(time_zone_offset_hour, digits);
}
else if (num_digits == 1)
{
readDecimalNumber<1>(time_zone_offset_hour, digits);
/// It looks like hhmmss
readDecimalNumber<2>(hour, digits);
readDecimalNumber<2>(minute, digits + 2);
readDecimalNumber<2>(second, digits + 4);
has_time = true;
}
else
return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME);
if (num_digits < 3 && checkChar(':', in))
{
num_digits = readDigits(digits, sizeof(digits), in);
/// It looks like time zone offset
has_time_zone_offset = true;
if (c == '-')
time_zone_offset_negative = true;
if (num_digits == 2)
if (num_digits == 4)
{
readDecimalNumber<2>(time_zone_offset_minute, digits);
readDecimalNumber<2>(time_zone_offset_hour, digits);
readDecimalNumber<2>(time_zone_offset_minute, digits + 2);
}
else if (num_digits == 3)
{
readDecimalNumber<1>(time_zone_offset_hour, digits);
readDecimalNumber<2>(time_zone_offset_minute, digits + 1);
}
else if (num_digits == 2)
{
readDecimalNumber<2>(time_zone_offset_hour, digits);
}
else if (num_digits == 1)
{
readDecimalNumber<1>(time_zone_offset_minute, digits);
readDecimalNumber<1>(time_zone_offset_hour, digits);
}
else
return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset in minutes: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME);
return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME);
if (num_digits < 3 && checkChar(':', in))
{
num_digits = readDigits(digits, sizeof(digits), in);
if (num_digits == 2)
{
readDecimalNumber<2>(time_zone_offset_minute, digits);
}
else if (num_digits == 1)
{
readDecimalNumber<1>(time_zone_offset_minute, digits);
}
else
return on_error("Cannot read DateTime: unexpected number of decimal digits for time zone offset in minutes: " + toString(num_digits), ErrorCodes::CANNOT_PARSE_DATETIME);
}
}
}
else

16
src/Interpreters/Access/InterpreterShowGrantsQuery.cpp
View File

@ -4,6 +4,8 @@
#include <Parsers/Access/ASTShowGrantsQuery.h>
#include <Parsers/formatAST.h>
#include <Access/AccessControl.h>
#include <Access/CachedAccessChecking.h>
#include <Access/ContextAccess.h>
#include <Access/Role.h>
#include <Access/RolesOrUsersSet.h>
#include <Access/User.h>
@ -135,15 +137,25 @@ QueryPipeline InterpreterShowGrantsQuery::executeImpl()
std::vector<AccessEntityPtr> InterpreterShowGrantsQuery::getEntities() const
{
const auto & show_query = query_ptr->as<ASTShowGrantsQuery &>();
const auto & access = getContext()->getAccess();
const auto & access_control = getContext()->getAccessControl();
const auto & show_query = query_ptr->as<ASTShowGrantsQuery &>();
auto ids = RolesOrUsersSet{*show_query.for_roles, access_control, getContext()->getUserID()}.getMatchingIDs(access_control);
CachedAccessChecking show_users(access, AccessType::SHOW_USERS);
CachedAccessChecking show_roles(access, AccessType::SHOW_ROLES);
bool throw_if_access_denied = !show_query.for_roles->all;
std::vector<AccessEntityPtr> entities;
for (const auto & id : ids)
{
auto entity = access_control.tryRead(id);
if (entity)
if (!entity)
continue;
if ((id == access->getUserID() /* Any user can see his own grants */)
|| (entity->isTypeOf<User>() && show_users.checkAccess(throw_if_access_denied))
|| (entity->isTypeOf<Role>() && show_roles.checkAccess(throw_if_access_denied)))
entities.push_back(entity);
}

15
src/Interpreters/AsynchronousInsertQueue.cpp
View File

@ -24,6 +24,16 @@
#include <base/logger_useful.h>
namespace CurrentMetrics
{
extern const Metric PendingAsyncInsert;
}
namespace ProfileEvents
{
extern const Event AsyncInsertQuery;
}
namespace DB
{
@ -223,6 +233,9 @@ void AsynchronousInsertQueue::pushImpl(InsertData::EntryPtr entry, QueueIterator
if (data->size > max_data_size)
scheduleDataProcessingJob(it->first, std::move(data), getContext());
CurrentMetrics::add(CurrentMetrics::PendingAsyncInsert);
ProfileEvents::increment(ProfileEvents::AsyncInsertQuery);
}
void AsynchronousInsertQueue::waitForProcessingQuery(const String & query_id, const Milliseconds & timeout)
@ -437,6 +450,8 @@ try
for (const auto & entry : data->entries)
if (!entry->isFinished())
entry->finish();
CurrentMetrics::sub(CurrentMetrics::PendingAsyncInsert, data->entries.size());
}
catch (const Exception & e)
{

12
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -1945,6 +1945,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
{
query_info.projection->order_optimizer = std::make_shared<ReadInOrderOptimizer>(
// TODO Do we need a projection variant for this field?
query,
analysis_result.order_by_elements_actions,
getSortDescription(query, context),
query_info.syntax_analyzer_result);
@ -1952,7 +1953,10 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
else
{
query_info.order_optimizer = std::make_shared<ReadInOrderOptimizer>(
analysis_result.order_by_elements_actions, getSortDescription(query, context), query_info.syntax_analyzer_result);
query,
analysis_result.order_by_elements_actions,
getSortDescription(query, context),
query_info.syntax_analyzer_result);
}
}
else
@ -1960,6 +1964,7 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
if (query_info.projection)
{
query_info.projection->order_optimizer = std::make_shared<ReadInOrderOptimizer>(
query,
query_info.projection->group_by_elements_actions,
getSortDescriptionFromGroupBy(query),
query_info.syntax_analyzer_result);
@ -1967,7 +1972,10 @@ void InterpreterSelectQuery::executeFetchColumns(QueryProcessingStage::Enum proc
else
{
query_info.order_optimizer = std::make_shared<ReadInOrderOptimizer>(
analysis_result.group_by_elements_actions, getSortDescriptionFromGroupBy(query), query_info.syntax_analyzer_result);
query,
analysis_result.group_by_elements_actions,
getSortDescriptionFromGroupBy(query),
query_info.syntax_analyzer_result);
}
}

45
src/Interpreters/OpenTelemetrySpanLog.cpp
View File

@ -150,24 +150,6 @@ OpenTelemetrySpanHolder::~OpenTelemetrySpanHolder()
}
}
template <typename T>
static T readHex(const char * data)
{
T x{};
const char * end = data + sizeof(T) * 2;
while (data < end)
{
x *= 16;
x += unhex(*data);
++data;
}
return x;
}
bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & traceparent,
std::string & error)
{
@ -185,7 +167,7 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace
const char * data = traceparent.data();
uint8_t version = readHex<uint8_t>(data);
uint8_t version = unhex2(data);
data += 2;
if (version != 0)
@ -201,7 +183,8 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace
}
++data;
UInt128 trace_id_128 = readHex<UInt128>(data);
UInt64 trace_id_higher_64 = unhexUInt<UInt64>(data);
UInt64 trace_id_lower_64 = unhexUInt<UInt64>(data + 16);
data += 32;
if (*data != '-')
@ -211,7 +194,7 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace
}
++data;
UInt64 span_id_64 = readHex<UInt64>(data);
UInt64 span_id_64 = unhexUInt<UInt64>(data);
data += 16;
if (*data != '-')
@ -221,8 +204,11 @@ bool OpenTelemetryTraceContext::parseTraceparentHeader(const std::string & trace
}
++data;
this->trace_flags = readHex<UInt8>(data);
this->trace_id = trace_id_128;
this->trace_flags = unhex2(data);
// store the 128-bit trace id in big-endian order
this->trace_id.toUnderType().items[0] = trace_id_higher_64;
this->trace_id.toUnderType().items[1] = trace_id_lower_64;
this->span_id = span_id_64;
return true;
}
@ -232,11 +218,14 @@ std::string OpenTelemetryTraceContext::composeTraceparentHeader() const
{
// This span is a parent for its children, so we specify this span_id as a
// parent id.
return fmt::format("00-{:032x}-{:016x}-{:02x}", __uint128_t(trace_id.toUnderType()),
span_id,
// This cast is needed because fmt is being weird and complaining that
// "mixing character types is not allowed".
static_cast<uint8_t>(trace_flags));
return fmt::format("00-{:016x}{:016x}-{:016x}-{:02x}",
// Output the trace id in network byte order
trace_id.toUnderType().items[0],
trace_id.toUnderType().items[1],
span_id,
// This cast is needed because fmt is being weird and complaining that
// "mixing character types is not allowed".
static_cast<uint8_t>(trace_flags));
}

6
src/Interpreters/ProcessList.cpp
View File

@ -17,6 +17,11 @@
#include <chrono>
namespace CurrentMetrics
{
extern const Metric Query;
}
namespace DB
{
@ -313,6 +318,7 @@ QueryStatus::QueryStatus(
, client_info(client_info_)
, priority_handle(std::move(priority_handle_))
, query_kind(query_kind_)
, num_queries_increment(CurrentMetrics::Query)
{
auto settings = getContext()->getSettings();
limits.max_execution_time = settings.max_execution_time;

4
src/Interpreters/ProcessList.h
View File

@ -121,6 +121,10 @@ protected:
IAST::QueryKind query_kind;
/// This field is unused in this class, but it
/// increments/decrements metric in constructor/destructor.
CurrentMetrics::Increment num_queries_increment;
public:
QueryStatus(

2
src/Interpreters/TreeCNFConverter.cpp
View File

@ -27,7 +27,7 @@ bool isLogicalFunction(const ASTFunction & func)
size_t countAtoms(const ASTPtr & node)
{
checkStackSize();
if (node->as<ASTIdentifier>())
if (node->as<ASTIdentifier>() || node->as<ASTLiteral>())
return 1;
const auto * func = node->as<ASTFunction>();

2
src/Interpreters/executeDDLQueryOnCluster.cpp
View File

@ -328,7 +328,7 @@ Chunk DDLQueryStatusSource::generate()
return {};
}
LOG_INFO(log, msg_format, node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts);
LOG_INFO(log, fmt::runtime(msg_format), node_path, timeout_seconds, num_unfinished_hosts, num_active_hosts);
NameSet unfinished_hosts = waiting_hosts;
for (const auto & host_id : finished_hosts)

2
src/Parsers/DumpASTNode.h
View File

@ -102,7 +102,7 @@ public:
~DebugASTLog()
{
if constexpr (_enable)
LOG_DEBUG(log, buf.str());
LOG_DEBUG(log, fmt::runtime(buf.str()));
}
WriteBuffer * stream() { return (_enable ? &buf : nullptr); }

19
src/Parsers/Lexer.cpp
View File

@ -280,6 +280,18 @@ Token Lexer::nextTokenImpl()
}
return Token(TokenType::Slash, token_begin, pos);
}
case '#': /// start of single line comment, MySQL style
{ /// PostgreSQL has some operators using '#' character.
/// For less ambiguity, we will recognize a comment only if # is followed by whitespace.
/// or #! as a special case for "shebang".
/// #hello - not a comment
/// # hello - a comment
/// #!/usr/bin/clickhouse-local --queries-file - a comment
++pos;
if (pos < end && (*pos == ' ' || *pos == '!'))
return comment_until_end_of_line();
return Token(TokenType::Error, token_begin, pos);
}
case '%':
return Token(TokenType::Percent, token_begin, ++pos);
case '=': /// =, ==
@ -335,6 +347,13 @@ Token Lexer::nextTokenImpl()
return Token(TokenType::DoubleAt, token_begin, ++pos);
return Token(TokenType::At, token_begin, pos);
}
case '\\':
{
++pos;
if (pos < end && *pos == 'G')
return Token(TokenType::VerticalDelimiter, token_begin, ++pos);
return Token(TokenType::Error, token_begin, pos);
}
default:
if (*pos == '$')

1
src/Parsers/Lexer.h
View File

@ -28,6 +28,7 @@ namespace DB
\
M(Comma) \
M(Semicolon) \
M(VerticalDelimiter) /** Vertical delimiter \G */ \
M(Dot) /** Compound identifiers, like a.b or tuple access operator a.1, (x, y).2. */ \
/** Need to be distinguished from floating point number with omitted integer part: .1 */ \
\

38
src/Processors/Formats/Impl/RegexpRowInputFormat.cpp
View File

@ -37,7 +37,7 @@ bool RegexpFieldExtractor::parseRow(PeekableReadBuffer & buf)
do
{
char * pos = find_first_symbols<'\n', '\r'>(buf.position(), buf.buffer().end());
char * pos = find_first_symbols<'\n'>(buf.position(), buf.buffer().end());
line_size += pos - buf.position();
buf.position() = pos;
} while (buf.position() == buf.buffer().end() && !buf.eof());
@ -45,15 +45,19 @@ bool RegexpFieldExtractor::parseRow(PeekableReadBuffer & buf)
buf.makeContinuousMemoryFromCheckpointToPos();
buf.rollbackToCheckpoint();
bool match = re2_st::RE2::FullMatchN(re2_st::StringPiece(buf.position(), line_size), regexp, re2_arguments_ptrs.data(), re2_arguments_ptrs.size());
/// Allow DOS line endings.
size_t line_to_match = line_size;
if (line_size > 0 && buf.position()[line_size - 1] == '\r')
--line_to_match;
bool match = re2_st::RE2::FullMatchN(re2_st::StringPiece(buf.position(), line_to_match), regexp, re2_arguments_ptrs.data(), re2_arguments_ptrs.size());
if (!match && !skip_unmatched)
throw Exception("Line \"" + std::string(buf.position(), line_size) + "\" doesn't match the regexp.", ErrorCodes::INCORRECT_DATA);
throw Exception("Line \"" + std::string(buf.position(), line_to_match) + "\" doesn't match the regexp.", ErrorCodes::INCORRECT_DATA);
buf.position() += line_size;
checkChar('\r', buf);
if (!buf.eof() && !checkChar('\n', buf))
throw Exception("No \\n after \\r at the end of line.", ErrorCodes::INCORRECT_DATA);
throw Exception("No \\n at the end of line.", ErrorCodes::LOGICAL_ERROR);
return match;
}
@ -65,12 +69,12 @@ RegexpRowInputFormat::RegexpRowInputFormat(
}
RegexpRowInputFormat::RegexpRowInputFormat(
std::unique_ptr<PeekableReadBuffer> buf_, const Block & header_, Params params_, const FormatSettings & format_settings_)
: IRowInputFormat(header_, *buf_, std::move(params_))
, buf(std::move(buf_))
, format_settings(format_settings_)
, escaping_rule(format_settings_.regexp.escaping_rule)
, field_extractor(RegexpFieldExtractor(format_settings_))
std::unique_ptr<PeekableReadBuffer> buf_, const Block & header_, Params params_, const FormatSettings & format_settings_)
: IRowInputFormat(header_, *buf_, std::move(params_))
, buf(std::move(buf_))
, format_settings(format_settings_)
, escaping_rule(format_settings_.regexp.escaping_rule)
, field_extractor(RegexpFieldExtractor(format_settings_))
{
}
@ -174,20 +178,12 @@ static std::pair<bool, size_t> fileSegmentationEngineRegexpImpl(ReadBuffer & in,
while (loadAtPosition(in, memory, pos) && need_more_data)
{
pos = find_first_symbols<'\n', '\r'>(pos, in.buffer().end());
pos = find_first_symbols<'\n'>(pos, in.buffer().end());
if (pos > in.buffer().end())
throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR);
throw Exception("Position in buffer is out of bounds. There must be a bug.", ErrorCodes::LOGICAL_ERROR);
else if (pos == in.buffer().end())
continue;
// Support DOS-style newline ("\r\n")
if (*pos == '\r')
{
++pos;
if (pos == in.buffer().end())
loadAtPosition(in, memory, pos);
}
if (memory.size() + static_cast<size_t>(pos - in.position()) >= min_chunk_size)
need_more_data = false;

31
src/Processors/QueryPlan/ReadFromMergeTree.cpp
View File

@ -367,7 +367,6 @@ static ActionsDAGPtr createProjection(const Block & header)
Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
RangesInDataParts && parts_with_ranges,
const Names & column_names,
const ActionsDAGPtr & sorting_key_prefix_expr,
ActionsDAGPtr & out_projection,
const InputOrderInfoPtr & input_order_info)
{
@ -509,10 +508,19 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsWithOrder(
if (need_preliminary_merge)
{
size_t fixed_prefix_size = input_order_info->order_key_fixed_prefix_descr.size();
size_t prefix_size = fixed_prefix_size + input_order_info->order_key_prefix_descr.size();
auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone();
order_key_prefix_ast->children.resize(prefix_size);
auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical());
auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false);
const auto & sorting_columns = metadata_snapshot->getSortingKey().column_names;
SortDescription sort_description;
for (size_t j = 0; j < input_order_info->order_key_prefix_descr.size(); ++j)
sort_description.emplace_back(metadata_snapshot->getSortingKey().column_names[j],
input_order_info->direction, 1);
for (size_t j = 0; j < prefix_size; ++j)
sort_description.emplace_back(sorting_columns[j], input_order_info->direction);
auto sorting_key_expr = std::make_shared<ExpressionActions>(sorting_key_prefix_expr);
@ -912,6 +920,11 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
parts_before_pk = parts.size();
auto reader_settings = getMergeTreeReaderSettings(context);
bool use_skip_indexes = context->getSettings().use_skip_indexes;
if (select.final() && !context->getSettings().use_skip_indexes_if_final)
use_skip_indexes = false;
result.parts_with_ranges = MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes(
std::move(parts),
metadata_snapshot,
@ -922,7 +935,7 @@ MergeTreeDataSelectAnalysisResultPtr ReadFromMergeTree::selectRangesToRead(
log,
num_streams,
result.index_stats,
context->getSettings().use_skip_indexes);
use_skip_indexes);
}
catch (...)
{
@ -1050,17 +1063,9 @@ void ReadFromMergeTree::initializePipeline(QueryPipelineBuilder & pipeline, cons
}
else if ((settings.optimize_read_in_order || settings.optimize_aggregation_in_order) && input_order_info)
{
size_t prefix_size = input_order_info->order_key_prefix_descr.size();
auto order_key_prefix_ast = metadata_snapshot->getSortingKey().expression_list_ast->clone();
order_key_prefix_ast->children.resize(prefix_size);
auto syntax_result = TreeRewriter(context).analyze(order_key_prefix_ast, metadata_snapshot->getColumns().getAllPhysical());
auto sorting_key_prefix_expr = ExpressionAnalyzer(order_key_prefix_ast, syntax_result, context).getActionsDAG(false);
pipe = spreadMarkRangesAmongStreamsWithOrder(
std::move(result.parts_with_ranges),
column_names_to_read,
sorting_key_prefix_expr,
result_projection,
input_order_info);
}

1
src/Processors/QueryPlan/ReadFromMergeTree.h
View File

@ -173,7 +173,6 @@ private:
Pipe spreadMarkRangesAmongStreamsWithOrder(
RangesInDataParts && parts_with_ranges,
const Names & column_names,
const ActionsDAGPtr & sorting_key_prefix_expr,
ActionsDAGPtr & out_projection,
const InputOrderInfoPtr & input_order_info);

4
src/Server/GRPCServer.cpp
View File

@ -754,7 +754,7 @@ namespace
// Parse the OpenTelemetry traceparent header.
ClientInfo client_info = session->getClientInfo();
auto & client_metadata = responder->grpc_context.client_metadata();
const auto & client_metadata = responder->grpc_context.client_metadata();
auto traceparent = client_metadata.find("traceparent");
if (traceparent != client_metadata.end())
{
@ -1262,7 +1262,7 @@ namespace
{
io.onException();
LOG_ERROR(log, getExceptionMessage(exception, true));
LOG_ERROR(log, fmt::runtime(getExceptionMessage(exception, true)));
if (responder && !responder_finished)
{

5
src/Server/HTTPHandler.cpp
View File

@ -915,7 +915,10 @@ void HTTPHandler::handleRequest(HTTPServerRequest & request, HTTPServerResponse
}
processQuery(request, params, response, used_output, query_scope);
LOG_DEBUG(log, (request_credentials ? "Authentication in progress..." : "Done processing query"));
if (request_credentials)
LOG_DEBUG(log, "Authentication in progress...");
else
LOG_DEBUG(log, "Done processing query");
}
catch (...)
{

6
src/Server/InterserverIOHTTPHandler.cpp
View File

@ -138,9 +138,9 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe
write_response(message);
if (is_real_error)
LOG_ERROR(log, message);
LOG_ERROR(log, fmt::runtime(message));
else
LOG_INFO(log, message);
LOG_INFO(log, fmt::runtime(message));
}
catch (...)
{
@ -148,7 +148,7 @@ void InterserverIOHTTPHandler::handleRequest(HTTPServerRequest & request, HTTPSe
std::string message = getCurrentExceptionMessage(false);
write_response(message);
LOG_ERROR(log, message);
LOG_ERROR(log, fmt::runtime(message));
}
}

6
src/Server/PostgreSQLHandler.cpp
View File

@ -105,7 +105,7 @@ void PostgreSQLHandler::run()
"0A000",
"Command is not supported"),
true);
LOG_ERROR(log, Poco::format("Command is not supported. Command code %d", static_cast<Int32>(message_type)));
LOG_ERROR(log, "Command is not supported. Command code {:d}", static_cast<Int32>(message_type));
message_transport->dropMessage();
}
}
@ -222,7 +222,7 @@ void PostgreSQLHandler::cancelRequest()
std::unique_ptr<PostgreSQLProtocol::Messaging::CancelRequest> msg =
message_transport->receiveWithPayloadSize<PostgreSQLProtocol::Messaging::CancelRequest>(8);
String query = Poco::format("KILL QUERY WHERE query_id = 'postgres:%d:%d'", msg->process_id, msg->secret_key);
String query = fmt::format("KILL QUERY WHERE query_id = 'postgres:{:d}:{:d}'", msg->process_id, msg->secret_key);
ReadBufferFromString replacement(query);
auto query_context = session->makeQueryContext();
@ -287,7 +287,7 @@ void PostgreSQLHandler::processQuery()
{
secret_key = dis(gen);
auto query_context = session->makeQueryContext();
query_context->setCurrentQueryId(Poco::format("postgres:%d:%d", connection_id, secret_key));
query_context->setCurrentQueryId(fmt::format("postgres:{:d}:{:d}", connection_id, secret_key));
CurrentThread::QueryScope query_scope{query_context};
ReadBufferFromString read_buf(spl_query);

8
src/Server/TCPHandler.cpp
View File

@ -466,7 +466,7 @@ void TCPHandler::runImpl()
}
const auto & e = *exception;
LOG_ERROR(log, getExceptionMessage(e, true));
LOG_ERROR(log, fmt::runtime(getExceptionMessage(e, true)));
sendException(*exception, send_exception_with_stack_trace);
}
}
@ -1392,6 +1392,12 @@ void TCPHandler::receiveQuery()
if (is_interserver_mode)
{
ClientInfo original_session_client_info = session->getClientInfo();
/// Cleanup fields that should not be reused from previous query.
original_session_client_info.current_user.clear();
original_session_client_info.current_query_id.clear();
original_session_client_info.current_address = {};
session = std::make_unique<Session>(server.context(), ClientInfo::Interface::TCP_INTERSERVER);
session->getClientInfo() = original_session_client_info;
}

44
src/Storages/MergeTree/DataPartsExchange.cpp
View File

@ -222,7 +222,7 @@ void Service::sendPartFromMemory(
auto projection_sample_block = metadata_snapshot->projections.get(name).sample_block;
auto part_in_memory = asInMemoryPart(projection);
if (!part_in_memory)
throw Exception("Projection " + name + " of part " + part->name + " is not stored in memory", ErrorCodes::LOGICAL_ERROR);
throw Exception(ErrorCodes::LOGICAL_ERROR, "Projection {} of part {} is not stored in memory", name, part->name);
writeStringBinary(name, out);
projection->checksums.write(out);
@ -232,7 +232,7 @@ void Service::sendPartFromMemory(
auto part_in_memory = asInMemoryPart(part);
if (!part_in_memory)
throw Exception("Part " + part->name + " is not stored in memory", ErrorCodes::LOGICAL_ERROR);
throw Exception(ErrorCodes::LOGICAL_ERROR, "Part {} is not stored in memory", part->name);
NativeWriter block_out(out, 0, metadata_snapshot->getSampleBlock());
part->checksums.write(out);
@ -300,7 +300,7 @@ MergeTreeData::DataPart::Checksums Service::sendPartFromDisk(
throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED);
if (hashing_out.count() != size)
throw Exception("Unexpected size of file " + path, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}", path);
writePODBinary(hashing_out.getHash(), out);
@ -323,7 +323,7 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part
auto disk = part->volume->getDisk();
if (!disk->supportZeroCopyReplication())
throw Exception(fmt::format("disk {} doesn't support zero-copy replication", disk->getName()), ErrorCodes::LOGICAL_ERROR);
throw Exception(ErrorCodes::LOGICAL_ERROR, "disk {} doesn't support zero-copy replication", disk->getName());
part->storage.lockSharedData(*part);
@ -340,9 +340,9 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part
fs::path metadata(metadata_file);
if (!fs::exists(metadata))
throw Exception("Remote metadata '" + file_name + "' is not exists", ErrorCodes::CORRUPTED_DATA);
throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not exists", file_name);
if (!fs::is_regular_file(metadata))
throw Exception("Remote metadata '" + file_name + "' is not a file", ErrorCodes::CORRUPTED_DATA);
throw Exception(ErrorCodes::CORRUPTED_DATA, "Remote metadata '{}' is not a file", file_name);
UInt64 file_size = fs::file_size(metadata);
writeStringBinary(it.first, out);
@ -355,7 +355,7 @@ void Service::sendPartFromDiskRemoteMeta(const MergeTreeData::DataPartPtr & part
throw Exception("Transferring part to replica was cancelled", ErrorCodes::ABORTED);
if (hashing_out.count() != file_size)
throw Exception("Unexpected size of file " + metadata_file, ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART);
throw Exception(ErrorCodes::BAD_SIZE_OF_FILE_IN_DATA_PART, "Unexpected size of file {}", metadata_file);
writePODBinary(hashing_out.getHash(), out);
}
@ -370,7 +370,7 @@ MergeTreeData::DataPartPtr Service::findPart(const String & name)
if (part)
return part;
throw Exception("No part " + name + " in table", ErrorCodes::NO_SUCH_DATA_PART);
throw Exception(ErrorCodes::NO_SUCH_DATA_PART, "No part {} in table", name);
}
MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
@ -511,9 +511,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
if (!try_zero_copy)
throw Exception("Got unexpected 'remote_fs_metadata' cookie", ErrorCodes::LOGICAL_ERROR);
if (std::find(capability.begin(), capability.end(), remote_fs_metadata) == capability.end())
throw Exception(fmt::format("Got 'remote_fs_metadata' cookie {}, expect one from {}", remote_fs_metadata, fmt::join(capability, ", ")), ErrorCodes::LOGICAL_ERROR);
throw Exception(ErrorCodes::LOGICAL_ERROR, "Got 'remote_fs_metadata' cookie {}, expect one from {}", remote_fs_metadata, fmt::join(capability, ", "));
if (server_protocol_version < REPLICATION_PROTOCOL_VERSION_WITH_PARTS_ZERO_COPY)
throw Exception(fmt::format("Got 'remote_fs_metadata' cookie with old protocol version {}", server_protocol_version), ErrorCodes::LOGICAL_ERROR);
throw Exception(ErrorCodes::LOGICAL_ERROR, "Got 'remote_fs_metadata' cookie with old protocol version {}", server_protocol_version);
if (part_type == "InMemory")
throw Exception("Got 'remote_fs_metadata' cookie for in-memory part", ErrorCodes::INCORRECT_PART_TYPE);
@ -525,7 +525,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::fetchPart(
{
if (e.code() != ErrorCodes::S3_ERROR && e.code() != ErrorCodes::ZERO_COPY_REPLICATION_ERROR)
throw;
LOG_WARNING(log, e.message() + " Will retry fetching part without zero-copy.");
LOG_WARNING(log, fmt::runtime(e.message() + " Will retry fetching part without zero-copy."));
/// Try again but without zero-copy
return fetchPart(metadata_snapshot, context, part_name, replica_path, host, port, timeouts,
user, password, interserver_scheme, throttler, to_detached, tmp_prefix_, nullptr, false, disk);
@ -649,9 +649,10 @@ void Fetcher::downloadBaseOrProjectionPartToDisk(
/// Otherwise malicious ClickHouse replica may force us to write to arbitrary path.
String absolute_file_path = fs::weakly_canonical(fs::path(part_download_path) / file_name);
if (!startsWith(absolute_file_path, fs::weakly_canonical(part_download_path).string()))
throw Exception("File path (" + absolute_file_path + ") doesn't appear to be inside part path (" + part_download_path + ")."
" This may happen if we are trying to download part from malicious replica or logical error.",
ErrorCodes::INSECURE_PATH);
throw Exception(ErrorCodes::INSECURE_PATH,
"File path ({}) doesn't appear to be inside part path ({}). "
"This may happen if we are trying to download part from malicious replica or logical error.",
absolute_file_path, part_download_path);
auto file_out = disk->writeFile(fs::path(part_download_path) / file_name);
HashingWriteBuffer hashing_out(*file_out);
@ -670,8 +671,10 @@ void Fetcher::downloadBaseOrProjectionPartToDisk(
readPODBinary(expected_hash, in);
if (expected_hash != hashing_out.getHash())
throw Exception("Checksum mismatch for file " + fullPath(disk, (fs::path(part_download_path) / file_name).string()) + " transferred from " + replica_path,
ErrorCodes::CHECKSUM_DOESNT_MATCH);
throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH,
"Checksum mismatch for file {} transferred from {}",
fullPath(disk, (fs::path(part_download_path) / file_name).string()),
replica_path);
if (file_name != "checksums.txt" &&
file_name != "columns.txt" &&
@ -762,7 +765,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta(
if (!disk->supportZeroCopyReplication() || !disk->checkUniqueId(part_id))
{
throw Exception(fmt::format("Part {} unique id {} doesn't exist on {}.", part_name, part_id, disk->getName()), ErrorCodes::ZERO_COPY_REPLICATION_ERROR);
throw Exception(ErrorCodes::ZERO_COPY_REPLICATION_ERROR, "Part {} unique id {} doesn't exist on {}.", part_name, part_id, disk->getName());
}
LOG_DEBUG(log, "Downloading Part {} unique id {} metadata onto disk {}.",
part_name, part_id, disk->getName());
@ -774,7 +777,7 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta(
String part_download_path = fs::path(data.getRelativeDataPath()) / part_relative_path / "";
if (disk->exists(part_download_path))
throw Exception("Directory " + fullPath(disk, part_download_path) + " already exists.", ErrorCodes::DIRECTORY_ALREADY_EXISTS);
throw Exception(ErrorCodes::DIRECTORY_ALREADY_EXISTS, "Directory {} already exists.", fullPath(disk, part_download_path));
CurrentMetrics::Increment metric_increment{CurrentMetrics::ReplicatedFetch};
@ -817,8 +820,9 @@ MergeTreeData::MutableDataPartPtr Fetcher::downloadPartToDiskRemoteMeta(
if (expected_hash != hashing_out.getHash())
{
throw Exception("Checksum mismatch for file " + metadata_file + " transferred from " + replica_path,
ErrorCodes::CHECKSUM_DOESNT_MATCH);
throw Exception(ErrorCodes::CHECKSUM_DOESNT_MATCH,
"Checksum mismatch for file {} transferred from {}",
metadata_file, replica_path);
}
}
}

4
src/Storages/MergeTree/IMergeTreeDataPart.cpp
View File

@ -1515,7 +1515,7 @@ void IMergeTreeDataPart::makeCloneOnDisk(const DiskPtr & disk, const String & di
if (disk->exists(fs::path(path_to_clone) / relative_path))
{
LOG_WARNING(storage.log, "Path " + fullPath(disk, path_to_clone + relative_path) + " already exists. Will remove it and clone again.");
LOG_WARNING(storage.log, "Path {} already exists. Will remove it and clone again.", fullPath(disk, path_to_clone + relative_path));
disk->removeRecursive(fs::path(path_to_clone) / relative_path / "");
}
disk->createDirectories(path_to_clone);
@ -1731,7 +1731,7 @@ UInt32 IMergeTreeDataPart::getNumberOfRefereneces() const
}
String IMergeTreeDataPart::getZeroLevelPartBlockID(std::string_view token) const
String IMergeTreeDataPart::getZeroLevelPartBlockID(const std::string_view token) const
{
if (info.level != 0)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Trying to get block id for non zero level part {}", name);

2
src/Storages/MergeTree/MergeFromLogEntryTask.cpp
View File

@ -84,7 +84,7 @@ std::pair<bool, ReplicatedMergeMutateTaskBase::PartLogWriter> MergeFromLogEntryT
/// 3. We have two intersecting parts, both cover source_part_name. It's logical error.
/// TODO Why 1 and 2 can happen? Do we need more assertions here or somewhere else?
constexpr const char * message = "Part {} is covered by {} but should be merged into {}. This shouldn't happen often.";
LOG_WARNING(log, message, source_part_name, source_part_or_covering->name, entry.new_part_name);
LOG_WARNING(log, fmt::runtime(message), source_part_name, source_part_or_covering->name, entry.new_part_name);
if (!source_part_or_covering->info.contains(MergeTreePartInfo::fromPartName(entry.new_part_name, storage.format_version)))
throw Exception(ErrorCodes::LOGICAL_ERROR, message, source_part_name, source_part_or_covering->name, entry.new_part_name);
return {false, {}};

25
src/Storages/MergeTree/MergeTreeBackgroundExecutor.cpp
View File

@ -3,11 +3,19 @@
#include <algorithm>
#include <Common/setThreadName.h>
#include <Common/Exception.h>
#include <Storages/MergeTree/BackgroundJobsAssignee.h>
namespace DB
{
namespace ErrorCodes
{
extern const int ABORTED;
}
template <class Queue>
void MergeTreeBackgroundExecutor<Queue>::wait()
{
@ -86,12 +94,18 @@ void MergeTreeBackgroundExecutor<Queue>::routine(TaskRuntimeDataPtr item)
ALLOW_ALLOCATIONS_IN_SCOPE;
need_execute_again = item->task->executeStep();
}
catch (const Exception & e)
{
if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info.
LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false)));
else
tryLogCurrentException(__PRETTY_FUNCTION__);
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
if (need_execute_again)
{
std::lock_guard guard(mutex);
@ -118,7 +132,6 @@ void MergeTreeBackgroundExecutor<Queue>::routine(TaskRuntimeDataPtr item)
return;
}
{
std::lock_guard guard(mutex);
erase_from_active();
@ -132,12 +145,18 @@ void MergeTreeBackgroundExecutor<Queue>::routine(TaskRuntimeDataPtr item)
/// But it is rather safe, because we have try...catch block here, and another one in ThreadPool.
item->task->onCompleted();
}
catch (const Exception & e)
{
if (e.code() == ErrorCodes::ABORTED) /// Cancelled merging parts is not an error - log as info.
LOG_INFO(log, fmt::runtime(getCurrentExceptionMessage(false)));
else
tryLogCurrentException(__PRETTY_FUNCTION__);
}
catch (...)
{
tryLogCurrentException(__PRETTY_FUNCTION__);
}
/// We have to call reset() under a lock, otherwise a race is possible.
/// Imagine, that task is finally completed (last execution returned false),
/// we removed the task from both queues, but still have pointer.

3
src/Storages/MergeTree/MergeTreeBackgroundExecutor.h
View File

@ -159,7 +159,6 @@ template <class Queue>
class MergeTreeBackgroundExecutor final : public shared_ptr_helper<MergeTreeBackgroundExecutor<Queue>>
{
public:
MergeTreeBackgroundExecutor(
String name_,
size_t threads_count_,
@ -194,7 +193,6 @@ public:
void wait();
private:
String name;
size_t threads_count{0};
size_t max_tasks_count{0};
@ -210,6 +208,7 @@ private:
std::condition_variable has_tasks;
std::atomic_bool shutdown{false};
ThreadPool pool;
Poco::Logger * log = &Poco::Logger::get("MergeTreeBackgroundExecutor");
};
extern template class MergeTreeBackgroundExecutor<MergeMutateRuntimeQueue>;

6
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -1758,7 +1758,7 @@ size_t MergeTreeData::clearOldWriteAheadLogs()
auto min_max_block_number = MergeTreeWriteAheadLog::tryParseMinMaxBlockNumber(it->name());
if (min_max_block_number && is_range_on_disk(min_max_block_number->first, min_max_block_number->second))
{
LOG_DEBUG(log, "Removing from filesystem the outdated WAL file " + it->name());
LOG_DEBUG(log, "Removing from filesystem the outdated WAL file {}", it->name());
disk_ptr->removeFile(relative_data_path + it->name());
++cleared_count;
}
@ -2642,7 +2642,7 @@ bool MergeTreeData::renameTempPartAndReplace(
/// deduplication.
if (deduplication_log)
{
String block_id = part->getZeroLevelPartBlockID(deduplication_token);
const String block_id = part->getZeroLevelPartBlockID(deduplication_token);
auto res = deduplication_log->addPart(block_id, part_info);
if (!res.second)
{
@ -6147,7 +6147,7 @@ ReservationPtr MergeTreeData::balancedReservation(
writeCString("\nbalancer: \n", log_str);
for (const auto & [disk_name, per_disk_parts] : disk_parts_for_logging)
writeString(fmt::format(" {}: [{}]\n", disk_name, fmt::join(per_disk_parts, ", ")), log_str);
LOG_DEBUG(log, log_str.str());
LOG_DEBUG(log, fmt::runtime(log_str.str()));
if (ttl_infos)
reserved_space = tryReserveSpacePreferringTTLRules(

Some files were not shown because too many files have changed in this diff Show More