Merged with master

This commit is contained in:
Vxider 2022-06-04 17:22:22 +00:00
commit 0246a3f415
1267 changed files with 13229 additions and 8627 deletions

View File

@ -62,6 +62,7 @@ Checks: '*,
-google-build-using-namespace,
-google-readability-braces-around-statements,
-google-readability-casting,
-google-readability-function-size,
-google-readability-namespace-comments,
-google-readability-todo,

View File

@ -2911,7 +2911,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison (actions)
CHECK_NAME=Performance Comparison
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=4
@ -2949,7 +2949,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison (actions)
CHECK_NAME=Performance Comparison
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=4
@ -2987,7 +2987,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison (actions)
CHECK_NAME=Performance Comparison
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4
@ -3025,7 +3025,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison (actions)
CHECK_NAME=Performance Comparison
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4

View File

@ -81,7 +81,6 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
BUILD_NAME=coverity
CACHES_PATH=${{runner.temp}}/../ccaches
CHECK_NAME=ClickHouse build check (actions)
IMAGES_PATH=${{runner.temp}}/images_path
REPO_COPY=${{runner.temp}}/build_check/ClickHouse
TEMP_PATH=${{runner.temp}}/build_check
@ -99,13 +98,15 @@ jobs:
id: coverity-checkout
uses: actions/checkout@v2
with:
submodules: 'true'
fetch-depth: 0 # otherwise we will have no info about contributors
- name: Build
run: |
git -C "$GITHUB_WORKSPACE" submodule sync
git -C "$GITHUB_WORKSPACE" submodule update --depth=1 --init --jobs=10
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$CHECK_NAME" "$BUILD_NAME"
cd "$REPO_COPY/tests/ci" && python3 build_check.py "$BUILD_NAME"
- name: Upload Coverity Analysis
if: ${{ success() || failure() }}
run: |

View File

@ -3127,7 +3127,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison (actions)
CHECK_NAME=Performance Comparison
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=4
@ -3165,7 +3165,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison (actions)
CHECK_NAME=Performance Comparison
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=4
@ -3203,7 +3203,7 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison (actions)
CHECK_NAME=Performance Comparison
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4
@ -3241,7 +3241,159 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison (actions)
CHECK_NAME=Performance Comparison
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{ env.REPORTS_PATH }}
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonAarch0:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, func-tester-aarch64]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison Aarch64
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=0
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{ env.REPORTS_PATH }}
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonAarch1:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, func-tester-aarch64]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison Aarch64
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=1
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{ env.REPORTS_PATH }}
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonAarch2:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, func-tester-aarch64]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison Aarch64
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=2
RUN_BY_HASH_TOTAL=4
EOF
- name: Download json reports
uses: actions/download-artifact@v2
with:
path: ${{ env.REPORTS_PATH }}
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
- name: Performance Comparison
run: |
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci"
python3 performance_comparison_check.py "$CHECK_NAME"
- name: Cleanup
if: always()
run: |
# shellcheck disable=SC2046
docker kill $(docker ps -q) ||:
# shellcheck disable=SC2046
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr "$TEMP_PATH"
PerformanceComparisonAarch3:
needs: [BuilderDebAarch64]
runs-on: [self-hosted, func-tester-aarch64]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/performance_comparison
REPORTS_PATH=${{runner.temp}}/reports_dir
CHECK_NAME=Performance Comparison Aarch64
REPO_COPY=${{runner.temp}}/performance_comparison/ClickHouse
RUN_BY_HASH_NUM=3
RUN_BY_HASH_TOTAL=4
@ -3333,6 +3485,10 @@ jobs:
- PerformanceComparison1
- PerformanceComparison2
- PerformanceComparison3
- PerformanceComparisonAarch0
- PerformanceComparisonAarch1
- PerformanceComparisonAarch2
- PerformanceComparisonAarch3
- UnitTestsAsan
- UnitTestsTsan
- UnitTestsMsan

View File

@ -36,7 +36,7 @@ set_property(GLOBAL PROPERTY USE_FOLDERS ON)
# Check that submodules are present
if (NOT EXISTS "${ClickHouse_SOURCE_DIR}/contrib/sysroot/README.md")
message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init --recursive")
message (FATAL_ERROR "Submodules are not initialized. Run\n\tgit submodule update --init")
endif ()
# Take care to add prlimit in command line before ccache, or else ccache thinks that

View File

@ -13,3 +13,7 @@ ClickHouse® is an open-source column-oriented database management system that a
* [Code Browser (Woboq)](https://clickhouse.com/codebrowser/ClickHouse/index.html) with syntax highlight and navigation.
* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlight, powered by github.dev.
* [Contacts](https://clickhouse.com/company/#contact) can help to get your questions answered if there are any.
## Upcoming Events
* [ClickHouse Meetup Amsterdam (in-person and online)](https://www.meetup.com/clickhouse-netherlands-user-group/events/286017044/) on June 8th, 2022

View File

@ -17,15 +17,12 @@ set (SRCS
sleep.cpp
terminalColors.cpp
errnoToString.cpp
ReplxxLineReader.cpp
StringRef.cpp
safeExit.cpp
throwError.cpp
)
if (ENABLE_REPLXX)
list (APPEND SRCS ReplxxLineReader.cpp)
endif ()
if (USE_DEBUG_HELPERS)
get_target_property(MAGIC_ENUM_INCLUDE_DIR ch_contrib::magic_enum INTERFACE_INCLUDE_DIRECTORIES)
# CMake generator expression will do insane quoting when it encounters special character like quotes, spaces, etc.

View File

@ -45,14 +45,16 @@ std::string replxx_now_ms_str()
time_t t = ms.count() / 1000;
tm broken;
if (!localtime_r(&t, &broken))
{
return std::string();
}
return {};
static int const BUFF_SIZE(32);
char str[BUFF_SIZE];
strftime(str, BUFF_SIZE, "%Y-%m-%d %H:%M:%S.", &broken);
snprintf(str + sizeof("YYYY-mm-dd HH:MM:SS"), 5, "%03d", static_cast<int>(ms.count() % 1000));
if (strftime(str, BUFF_SIZE, "%Y-%m-%d %H:%M:%S.", &broken) <= 0)
return {};
if (snprintf(str + sizeof("YYYY-mm-dd HH:MM:SS"), 5, "%03d", static_cast<int>(ms.count() % 1000)) <= 0)
return {};
return str;
}

View File

@ -576,8 +576,8 @@ private:
else if constexpr (Bits == 128 && sizeof(base_type) == 8)
{
using CompilerUInt128 = unsigned __int128;
CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0];
CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0];
CompilerUInt128 a = (CompilerUInt128(lhs.items[1]) << 64) + lhs.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 b = (CompilerUInt128(rhs.items[1]) << 64) + rhs.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 c = a * b;
integer<Bits, Signed> res;
res.items[0] = c;
@ -841,8 +841,8 @@ public:
{
using CompilerUInt128 = unsigned __int128;
CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0];
CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0];
CompilerUInt128 a = (CompilerUInt128(numerator.items[1]) << 64) + numerator.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 b = (CompilerUInt128(denominator.items[1]) << 64) + denominator.items[0]; // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
CompilerUInt128 c = a / b; // NOLINT
integer<Bits, Signed> res;
@ -1204,7 +1204,7 @@ constexpr integer<Bits, Signed>::operator T() const noexcept
UnsignedT res{};
for (unsigned i = 0; i < _impl::item_count && i < (sizeof(T) + sizeof(base_type) - 1) / sizeof(base_type); ++i)
res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i);
res += UnsignedT(items[i]) << (sizeof(base_type) * 8 * i); // NOLINT(clang-analyzer-core.UndefinedBinaryOperatorResult)
return res;
}

View File

@ -1,7 +1,4 @@
set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt")
message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive")
endif()
set(BUILD_TESTING OFF)
set(ABSL_PROPAGATE_CXX_STD ON)
add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp")

View File

@ -5,6 +5,12 @@ if (NOT ENABLE_AMQPCPP)
return()
endif()
# can be removed once libuv build on MacOS with GCC is possible
if (NOT TARGET ch_contrib::uv)
message(STATUS "Not using AMQP-CPP because libuv is disabled")
return()
endif()
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/AMQP-CPP")
set (SRCS
@ -32,21 +38,6 @@ set (SRCS
add_library(_amqp-cpp ${SRCS})
target_compile_options (_amqp-cpp
PRIVATE
-Wno-old-style-cast
-Wno-inconsistent-missing-destructor-override
-Wno-deprecated
-Wno-unused-parameter
-Wno-shadow
-Wno-tautological-type-limit-compare
-Wno-extra-semi
# NOTE: disable all warnings at last because the warning:
# "conversion function converting 'XXX' to itself will never be used"
# doesn't have it's own diagnostic flag yet.
-w
)
target_include_directories (_amqp-cpp SYSTEM BEFORE PUBLIC "${LIBRARY_DIR}/include" "${LIBRARY_DIR}")
target_link_libraries (_amqp-cpp PUBLIC OpenSSL::Crypto OpenSSL::SSL ch_contrib::uv)
add_library (ch_contrib::amqp_cpp ALIAS _amqp-cpp)

View File

@ -20,7 +20,7 @@ endif()
option (ENABLE_PARQUET "Enable parquet" ${ENABLE_PARQUET_DEFAULT})
if (NOT ENABLE_PARQUET)
message(STATUS "Building without Parquet support")
message(STATUS "Not using parquet")
return()
endif()

View File

@ -60,14 +60,6 @@ target_compile_definitions (_avrocpp PUBLIC SNAPPY_CODEC_AVAILABLE)
target_include_directories (_avrocpp PRIVATE ${SNAPPY_INCLUDE_DIR})
target_link_libraries (_avrocpp PRIVATE ch_contrib::snappy)
if (COMPILER_GCC)
set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor)
elseif (COMPILER_CLANG)
set (SUPPRESS_WARNINGS -Wno-non-virtual-dtor)
endif ()
target_compile_options(_avrocpp PRIVATE ${SUPPRESS_WARNINGS})
# create a symlink to include headers with <avro/...>
set(AVRO_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/include")
ADD_CUSTOM_TARGET(avro_symlink_headers ALL

View File

@ -52,20 +52,6 @@ include("${AZURE_DIR}/cmake-modules/AzureTransportAdapters.cmake")
add_library(_azure_sdk ${AZURE_SDK_UNIFIED_SRC})
if (COMPILER_CLANG)
target_compile_options(_azure_sdk PRIVATE
-Wno-deprecated-copy-dtor
-Wno-extra-semi
-Wno-suggest-destructor-override
-Wno-inconsistent-missing-destructor-override
-Wno-error=unknown-warning-option
)
if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL 13)
target_compile_options(_azure_sdk PRIVATE -Wno-reserved-identifier)
endif()
endif()
# Originally, on Windows azure-core is built with bcrypt and crypt32 by default
if (TARGET OpenSSL::SSL)
target_link_libraries(_azure_sdk PRIVATE OpenSSL::Crypto OpenSSL::SSL)

View File

@ -1,7 +1,12 @@
# Needed for:
# - securely connecting to an external server, e.g. clickhouse-client --host ... --secure
# - lots of thirdparty libraries
option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
# Actually, so many 3rd party libraries + unit tests need SSL that we cannot disable it
# without breaking the build ...
option(ENABLE_SSL "Enable ssl" ON) # breaks if OFF
# TODO: Making SSL dependent on ENABLE_LIBRARIES is desirable but needs fixing dependent libs + tests.
# option(ENABLE_SSL "Enable ssl" ${ENABLE_LIBRARIES})
if(NOT ENABLE_SSL)
message(STATUS "Not using openssl")

View File

@ -45,7 +45,4 @@ add_library(ch_contrib::brotli ALIAS _brotli)
target_include_directories(_brotli SYSTEM BEFORE PUBLIC "${BROTLI_SOURCE_DIR}/include")
if(M_LIBRARY)
target_link_libraries(_brotli PRIVATE ${M_LIBRARY})
endif()
target_compile_definitions(_brotli PRIVATE BROTLI_BUILD_PORTABLE=1)

View File

@ -1,6 +1,6 @@
option(ENABLE_BZIP2 "Enable bzip2 compression support" ${ENABLE_LIBRARIES})
if (NOT ENABLE_BZIP2)
message (STATUS "bzip2 compression disabled")
message (STATUS "Not using bzip2")
return()
endif()
@ -26,8 +26,4 @@ configure_file (
add_library(_bzip2 ${SRCS})
add_library(ch_contrib::bzip2 ALIAS _bzip2)
# To avoid -Wreserved-id-macro we use SYSTEM:
#
# clickhouse/contrib/bzip2/bzlib.h:23:9: error: macro name is a reserved identifier [-Werror,-Wreserved-id-macro]
# #define _BZLIB_H
target_include_directories(_bzip2 SYSTEM BEFORE PUBLIC "${BZIP2_SOURCE_DIR}" "${BZIP2_BINARY_DIR}")

View File

@ -81,16 +81,12 @@ set (CAPNPC_SRCS
add_library(_capnpc ${CAPNPC_SRCS})
target_link_libraries(_capnpc PUBLIC _capnp)
# The library has substandard code
if (COMPILER_GCC)
set (SUPPRESS_WARNINGS -w)
elseif (COMPILER_CLANG)
set (SUPPRESS_WARNINGS -w)
if (COMPILER_CLANG)
set (CAPNP_PRIVATE_CXX_FLAGS -fno-char8_t)
endif ()
target_compile_options(_kj PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnp PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnpc PRIVATE ${SUPPRESS_WARNINGS} ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_kj PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnp PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
target_compile_options(_capnpc PRIVATE ${CAPNP_PRIVATE_CXX_FLAGS})
add_library(ch_contrib::capnp ALIAS _capnpc)

View File

@ -5,6 +5,12 @@ if (NOT ENABLE_CASSANDRA)
return()
endif()
# can be removed once libuv build on MacOS with GCC is possible
if (NOT TARGET ch_contrib::uv)
message(STATUS "Not using cassandra because libuv is disabled")
return()
endif()
if (APPLE)
set(CMAKE_MACOSX_RPATH ON)
endif()

View File

@ -1,5 +1,5 @@
if (NOT ENABLE_KAFKA)
message(STATUS "Not using librdkafka (skip cppkafka)")
message(STATUS "Not using kafka")
return()
endif()

View File

@ -5,7 +5,7 @@ elseif(ENABLE_FASTOPS)
endif()
if(NOT ENABLE_FASTOPS)
message(STATUS "Not using fast vectorized mathematical functions library by Mikhail Parakhin")
message(STATUS "Not using fastops")
return()
endif()

View File

@ -1,22 +1,24 @@
set(FMT_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/fmtlib")
set (SRCS
# NOTE: do not build module for now:
# ../fmtlib/src/fmt.cc
../fmtlib/src/format.cc
../fmtlib/src/os.cc
${FMT_SOURCE_DIR}/src/format.cc
${FMT_SOURCE_DIR}/src/os.cc
../fmtlib/include/fmt/args.h
../fmtlib/include/fmt/chrono.h
../fmtlib/include/fmt/color.h
../fmtlib/include/fmt/compile.h
../fmtlib/include/fmt/core.h
../fmtlib/include/fmt/format.h
../fmtlib/include/fmt/format-inl.h
../fmtlib/include/fmt/locale.h
../fmtlib/include/fmt/os.h
../fmtlib/include/fmt/ostream.h
../fmtlib/include/fmt/printf.h
../fmtlib/include/fmt/ranges.h
../fmtlib/include/fmt/xchar.h
${FMT_SOURCE_DIR}/include/fmt/args.h
${FMT_SOURCE_DIR}/include/fmt/chrono.h
${FMT_SOURCE_DIR}/include/fmt/color.h
${FMT_SOURCE_DIR}/include/fmt/compile.h
${FMT_SOURCE_DIR}/include/fmt/core.h
${FMT_SOURCE_DIR}/include/fmt/format.h
${FMT_SOURCE_DIR}/include/fmt/format-inl.h
${FMT_SOURCE_DIR}/include/fmt/locale.h
${FMT_SOURCE_DIR}/include/fmt/os.h
${FMT_SOURCE_DIR}/include/fmt/ostream.h
${FMT_SOURCE_DIR}/include/fmt/printf.h
${FMT_SOURCE_DIR}/include/fmt/ranges.h
${FMT_SOURCE_DIR}/include/fmt/xchar.h
)
add_library(_fmt ${SRCS})

View File

@ -9,23 +9,23 @@ set(H3_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/h3/src/h3lib")
set(H3_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/h3/src/h3lib")
set(SRCS
"${H3_SOURCE_DIR}/lib/algos.c"
"${H3_SOURCE_DIR}/lib/coordijk.c"
"${H3_SOURCE_DIR}/lib/bbox.c"
"${H3_SOURCE_DIR}/lib/polygon.c"
"${H3_SOURCE_DIR}/lib/h3Index.c"
"${H3_SOURCE_DIR}/lib/vec2d.c"
"${H3_SOURCE_DIR}/lib/vec3d.c"
"${H3_SOURCE_DIR}/lib/vertex.c"
"${H3_SOURCE_DIR}/lib/linkedGeo.c"
"${H3_SOURCE_DIR}/lib/localij.c"
"${H3_SOURCE_DIR}/lib/latLng.c"
"${H3_SOURCE_DIR}/lib/directedEdge.c"
"${H3_SOURCE_DIR}/lib/mathExtensions.c"
"${H3_SOURCE_DIR}/lib/iterators.c"
"${H3_SOURCE_DIR}/lib/vertexGraph.c"
"${H3_SOURCE_DIR}/lib/faceijk.c"
"${H3_SOURCE_DIR}/lib/baseCells.c"
"${H3_SOURCE_DIR}/lib/algos.c"
"${H3_SOURCE_DIR}/lib/coordijk.c"
"${H3_SOURCE_DIR}/lib/bbox.c"
"${H3_SOURCE_DIR}/lib/polygon.c"
"${H3_SOURCE_DIR}/lib/h3Index.c"
"${H3_SOURCE_DIR}/lib/vec2d.c"
"${H3_SOURCE_DIR}/lib/vec3d.c"
"${H3_SOURCE_DIR}/lib/vertex.c"
"${H3_SOURCE_DIR}/lib/linkedGeo.c"
"${H3_SOURCE_DIR}/lib/localij.c"
"${H3_SOURCE_DIR}/lib/latLng.c"
"${H3_SOURCE_DIR}/lib/directedEdge.c"
"${H3_SOURCE_DIR}/lib/mathExtensions.c"
"${H3_SOURCE_DIR}/lib/iterators.c"
"${H3_SOURCE_DIR}/lib/vertexGraph.c"
"${H3_SOURCE_DIR}/lib/faceijk.c"
"${H3_SOURCE_DIR}/lib/baseCells.c"
)
configure_file("${H3_SOURCE_DIR}/include/h3api.h.in" "${H3_BINARY_DIR}/include/h3api.h")
@ -34,8 +34,5 @@ add_library(_h3 ${SRCS})
target_include_directories(_h3 SYSTEM PUBLIC "${H3_SOURCE_DIR}/include")
target_include_directories(_h3 SYSTEM PUBLIC "${H3_BINARY_DIR}/include")
target_compile_definitions(_h3 PRIVATE H3_HAVE_VLA)
if(M_LIBRARY)
target_link_libraries(_h3 PRIVATE ${M_LIBRARY})
endif()
add_library(ch_contrib::h3 ALIAS _h3)

View File

@ -5,7 +5,7 @@ elseif(ENABLE_HIVE)
endif()
if (NOT ENABLE_HIVE)
message("Hive disabled")
message(STATUS "Not using hive")
return()
endif()

View File

@ -481,10 +481,6 @@ target_include_directories(_icui18n SYSTEM PUBLIC "${ICU_SOURCE_DIR}/i18n/")
target_compile_definitions(_icuuc PRIVATE -DU_COMMON_IMPLEMENTATION)
target_compile_definitions(_icui18n PRIVATE -DU_I18N_IMPLEMENTATION)
if (COMPILER_CLANG)
target_compile_options(_icudata PRIVATE -Wno-unused-command-line-argument)
endif ()
add_library(_icu INTERFACE)
target_link_libraries(_icu INTERFACE _icui18n _icuuc _icudata)
add_library(ch_contrib::icu ALIAS _icu)

View File

@ -180,7 +180,6 @@ if (USE_UNWIND)
target_link_libraries (_jemalloc PRIVATE unwind)
endif ()
target_compile_options(_jemalloc PRIVATE -Wno-redundant-decls)
# for RTLD_NEXT
target_compile_options(_jemalloc PRIVATE -D_GNU_SOURCE)

View File

@ -6,7 +6,7 @@ elseif(ENABLE_CPUID)
endif()
if (NOT ENABLE_CPUID)
message("Not using cpuid")
message(STATUS "Not using cpuid")
return()
endif()
@ -27,8 +27,5 @@ add_library (_cpuid ${SRCS})
target_include_directories (_cpuid SYSTEM PUBLIC "${LIBRARY_DIR}")
target_compile_definitions (_cpuid PRIVATE VERSION="v0.4.1")
if (COMPILER_CLANG)
target_compile_options (_cpuid PRIVATE -Wno-reserved-id-macro)
endif ()
add_library(ch_contrib::cpuid ALIAS _cpuid)

View File

@ -1,7 +1,7 @@
option(ENABLE_GSASL_LIBRARY "Enable gsasl library" ${ENABLE_LIBRARIES})
if (NOT ENABLE_GSASL_LIBRARY)
message(STATUS "Not using gsasl library")
message(STATUS "Not using gsasl")
return()
endif()

View File

@ -1,3 +1,4 @@
# once fixed, please remove similar places in CMakeLists of libuv users (search "ch_contrib::uv")
if (OS_DARWIN AND COMPILER_GCC)
message (WARNING "libuv cannot be built with GCC in macOS due to a bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=93082")
return()

2
contrib/libxml2 vendored

@ -1 +1 @@
Subproject commit a075d256fd9ff15590b86d981b75a50ead124fca
Subproject commit 7846b0a677f8d3ce72486125fa281e92ac9970e8

View File

@ -53,9 +53,6 @@ set(SRCS
add_library(_libxml2 ${SRCS})
target_link_libraries(_libxml2 PRIVATE ch_contrib::zlib)
if(M_LIBRARY)
target_link_libraries(_libxml2 PRIVATE ${M_LIBRARY})
endif()
target_include_directories(_libxml2 BEFORE PUBLIC "${CMAKE_CURRENT_SOURCE_DIR}/linux_x86_64/include")
target_include_directories(_libxml2 BEFORE PUBLIC "${LIBXML2_SOURCE_DIR}/include")

View File

@ -1,6 +1,6 @@
option(ENABLE_MINIZIP "Enable minizip-ng the zip manipulation library" ${ENABLE_LIBRARIES})
if (NOT ENABLE_MINIZIP)
message (STATUS "minizip-ng disabled")
message (STATUS "Not using minizip-ng")
return()
endif()

View File

@ -2,12 +2,12 @@ if (NOT ENABLE_ODBC)
return ()
endif ()
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc")
if (NOT TARGET ch_contrib::unixodbc)
message(FATAL_ERROR "Configuration error: unixodbc is not a target")
endif()
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/nanodbc")
set (SRCS
"${LIBRARY_DIR}/nanodbc/nanodbc.cpp"
)

View File

@ -1,10 +1,3 @@
option (ENABLE_REPLXX "Enable replxx support" ${ENABLE_LIBRARIES})
if (NOT ENABLE_REPLXX)
message (STATUS "Not using replxx")
return()
endif()
set (LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/replxx")
set(SRCS
@ -22,9 +15,4 @@ set(SRCS
add_library (_replxx ${SRCS})
target_include_directories(_replxx SYSTEM PUBLIC "${LIBRARY_DIR}/include")
if (COMPILER_CLANG)
target_compile_options(_replxx PRIVATE -Wno-documentation)
endif ()
add_library(ch_contrib::replxx ALIAS _replxx)

View File

@ -149,7 +149,3 @@ target_link_libraries(_s2 PRIVATE
target_include_directories(_s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/")
target_include_directories(_s2 SYSTEM PUBLIC "${ABSL_SOURCE_DIR}")
if(M_LIBRARY)
target_link_libraries(_s2 PRIVATE ${M_LIBRARY})
endif()

View File

@ -1,7 +1,7 @@
option(ENABLE_THRIFT "Enable Thrift" ${ENABLE_LIBRARIES})
if (NOT ENABLE_THRIFT)
message (STATUS "thrift disabled")
message (STATUS "Not using thrift")
return()
endif()

View File

@ -294,14 +294,6 @@ target_include_directories (_unixodbc
"${LIBRARY_DIR}/include"
)
target_compile_definitions (_unixodbc PRIVATE -DHAVE_CONFIG_H)
target_compile_options (_unixodbc
PRIVATE
-Wno-dangling-else
-Wno-parentheses
-Wno-misleading-indentation
-Wno-unknown-warning-option
-Wno-reserved-id-macro
-O2
)
target_compile_options (_unixodbc PRIVATE -O2) # intended?
add_library (ch_contrib::unixodbc ALIAS _unixodbc)

View File

@ -8,6 +8,8 @@ if [ "$GIT_DOCS_BRANCH" ] && ! [ "$GIT_DOCS_BRANCH" == "$GIT_BRANCH" ]; then
git fetch origin --depth=1 -- "$GIT_DOCS_BRANCH:$GIT_DOCS_BRANCH"
git checkout "$GIT_DOCS_BRANCH"
else
# Untracked yarn.lock could cause pull to fail
git clean -fdx
# Update docs repo
git pull
fi

View File

@ -6,7 +6,7 @@ FROM ubuntu:20.04
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
RUN apt-get update \
&& apt-get install \

View File

@ -319,25 +319,16 @@ if __name__ == "__main__":
)
parser.add_argument("--output-dir", type=dir_name, required=True)
parser.add_argument("--build-type", choices=("debug", ""), default="")
parser.add_argument(
"--compiler",
choices=(
"clang-11",
"clang-11-darwin",
"clang-11-darwin-aarch64",
"clang-11-aarch64",
"clang-12",
"clang-12-darwin",
"clang-12-darwin-aarch64",
"clang-12-aarch64",
"clang-13",
"clang-13-darwin",
"clang-13-darwin-aarch64",
"clang-13-aarch64",
"clang-13-ppc64le",
"clang-11-freebsd",
"clang-12-freebsd",
"clang-13-freebsd",
"clang-14",
"clang-14-darwin",
"clang-14-darwin-aarch64",
"clang-14-aarch64",
"clang-14-ppc64le",
"clang-14-freebsd",
"gcc-11",
),
default="clang-13",
@ -348,6 +339,7 @@ if __name__ == "__main__":
choices=("address", "thread", "memory", "undefined", ""),
default="",
)
parser.add_argument("--split-binary", action="store_true")
parser.add_argument("--clang-tidy", action="store_true")
parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")

View File

@ -7,7 +7,7 @@ FROM clickhouse/test-util:$FROM_TAG
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
RUN apt-get update \
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \

View File

@ -8,14 +8,18 @@ FROM clickhouse/binary-builder:$FROM_TAG
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-13 libllvm13 libclang-13-dev libmlir-13-dev
RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-14 libllvm14 libclang-14-dev libmlir-14-dev
# repo versions doesn't work correctly with C++17
# also we push reports to s3, so we add index.html to subfolder urls
# https://github.com/ClickHouse-Extras/woboq_codebrowser/commit/37e15eaf377b920acb0b48dbe82471be9203f76b
RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser
RUN cd woboq_codebrowser && cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-13 -DCMAKE_C_COMPILER=clang-13 && make -j
# TODO: remove branch in a few weeks after merge, e.g. in May or June 2022
RUN git clone https://github.com/ClickHouse-Extras/woboq_codebrowser --branch llvm-14 \
&& cd woboq_codebrowser \
&& cmake . -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_COMPILER=clang\+\+-14 -DCMAKE_C_COMPILER=clang-14 \
&& make -j \
&& cd .. \
&& rm -rf woboq_codebrowser
ENV CODEGEN=/woboq_codebrowser/generator/codebrowser_generator
ENV CODEINDEX=/woboq_codebrowser/indexgenerator/codebrowser_indexgenerator
@ -28,7 +32,7 @@ ENV SHA=nosha
ENV DATA="https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data"
CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \
cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-13 -DCMAKE_C_COMPILER=/usr/bin/clang-13 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-14 -DCMAKE_C_COMPILER=/usr/bin/clang-14 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
mkdir -p $HTML_RESULT_DIRECTORY && \
$CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \
cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\

View File

@ -7,7 +7,7 @@ FROM clickhouse/test-util:$FROM_TAG
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=13
ENV DEBIAN_FRONTEND=noninteractive LLVM_VERSION=14
RUN apt-get update \
&& apt-get install ca-certificates lsb-release wget gnupg apt-transport-https \

View File

@ -12,7 +12,7 @@ stage=${stage:-}
script_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
echo "$script_dir"
repo_dir=ch
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-13_debug_none_bundled_unsplitted_disable_False_binary"}
BINARY_TO_DOWNLOAD=${BINARY_TO_DOWNLOAD:="clang-14_debug_none_bundled_unsplitted_disable_False_binary"}
BINARY_URL_TO_DOWNLOAD=${BINARY_URL_TO_DOWNLOAD:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/$BINARY_TO_DOWNLOAD/clickhouse"}
function clone
@ -355,22 +355,8 @@ fi
cat > report.html <<EOF ||:
<!DOCTYPE html>
<html lang="en">
<link rel="preload" as="font" href="https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2" type="font/woff2" crossorigin="anonymous"/>
<style>
@font-face {
font-family:'Yandex Sans Display Web';
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
font-weight:400;
font-style:normal;
font-stretch:normal
}
body { font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }
body { font-family: "DejaVu Sans", "Noto Sans", Arial, sans-serif; background: #EEE; }
h1 { margin-left: 10px; }
th, td { border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
td { white-space: pre; font-family: Monospace, Courier New; }
@ -378,7 +364,6 @@ border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0,
a { color: #06F; text-decoration: none; }
a:hover, a:active { color: #F40; text-decoration: underline; }
table { border: 0; }
.main { margin-left: 10%; }
p.links a { padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }
th { cursor: pointer; }

View File

@ -2,7 +2,7 @@
set -euo pipefail
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-13_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
CLICKHOUSE_PACKAGE=${CLICKHOUSE_PACKAGE:="https://clickhouse-builds.s3.amazonaws.com/$PR_TO_TEST/$SHA_TO_TEST/clickhouse_build_check/clang-14_relwithdebuginfo_none_bundled_unsplitted_disable_False_binary/clickhouse"}
CLICKHOUSE_REPO_PATH=${CLICKHOUSE_REPO_PATH:=""}

View File

@ -207,6 +207,13 @@ function run_tests
test_files=($(ls "$test_prefix"/*.xml))
fi
# We can filter out certain tests
if [ -v CHPC_TEST_GREP_EXCLUDE ]; then
# filter tests array in bash https://stackoverflow.com/a/40375567
filtered_test_files=( $( for i in ${test_files[@]} ; do echo $i ; done | grep -v ${CHPC_TEST_GREP_EXCLUDE} ) )
test_files=("${filtered_test_files[@]}")
fi
# We split perf tests into multiple checks to make them faster
if [ -v CHPC_TEST_RUN_BY_HASH_TOTAL ]; then
# filter tests array in bash https://stackoverflow.com/a/40375567

View File

@ -17,8 +17,7 @@ ENV EXPORT_S3_STORAGE_POLICIES=1
# Download Minio-related binaries
RUN arch=${TARGETARCH:-amd64} \
&& wget "https://dl.min.io/server/minio/release/linux-${arch}/minio" \
&& chmod +x ./minio \
&& if [ "$arch" = "amd64" ] ; then wget "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio-20220103182258.0.0.x86_64.rpm"; else wget "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio-20220103182258.0.0.aarch64.rpm" ; fi \
&& wget "https://dl.min.io/client/mc/release/linux-${arch}/mc" \
&& chmod +x ./mc
ENV MINIO_ROOT_USER="clickhouse"

View File

@ -9,6 +9,10 @@
set -e -x -a -u
rpm2cpio ./minio-20220103182258.0.0.*.rpm | cpio -i --make-directories
find / -name minio
cp ./usr/local/bin/minio ./
ls -lha
mkdir -p ./minio_data
@ -27,12 +31,19 @@ fi
MINIO_ROOT_USER=${MINIO_ROOT_USER:-clickhouse}
MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-clickhouse}
./minio --version
./minio server --address ":11111" ./minio_data &
i=0
while ! curl -v --silent http://localhost:11111 2>&1 | grep AccessDenied
do
if [[ $i == 60 ]]; then
echo "Failed to setup minio"
exit 0
fi
echo "Trying to connect to minio"
sleep 1
i=$((i + 1))
done
lsof -i :11111

View File

@ -32,7 +32,11 @@ RUN apt-get update -y \
mysql-client=8.0* \
postgresql-client \
sqlite3 \
awscli
awscli \
openjdk-11-jre-headless \
rpm2cpio \
cpio
RUN pip3 install numpy scipy pandas Jinja2
@ -53,15 +57,20 @@ ARG TARGETARCH
# Download Minio-related binaries
RUN arch=${TARGETARCH:-amd64} \
&& wget "https://dl.min.io/server/minio/release/linux-${arch}/minio" \
&& chmod +x ./minio \
&& if [ "$arch" = "amd64" ] ; then wget "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio-20220103182258.0.0.x86_64.rpm"; else wget "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio-20220103182258.0.0.aarch64.rpm" ; fi \
&& wget "https://dl.min.io/client/mc/release/linux-${arch}/mc" \
&& chmod +x ./mc
RUN wget 'https://dlcdn.apache.org/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
&& tar -xvf hadoop-3.3.1.tar.gz \
&& rm -rf hadoop-3.3.1.tar.gz
ENV MINIO_ROOT_USER="clickhouse"
ENV MINIO_ROOT_PASSWORD="clickhouse"
ENV EXPORT_S3_STORAGE_POLICIES=1
COPY run.sh /
COPY setup_minio.sh /
COPY setup_hdfs_minicluster.sh /
CMD ["/bin/bash", "/run.sh"]

View File

@ -19,6 +19,7 @@ ln -s /usr/share/clickhouse-test/clickhouse-test /usr/bin/clickhouse-test
/usr/share/clickhouse-test/config/install.sh
./setup_minio.sh
./setup_hdfs_minicluster.sh
# For flaky check we also enable thread fuzzer
if [ "$NUM_TRIES" -gt "1" ]; then
@ -92,8 +93,6 @@ function run_tests()
if [[ -n "$USE_DATABASE_REPLICATED" ]] && [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
ADDITIONAL_OPTIONS+=('--replicated-database')
# Cannot be used with replicated database, due to distributed_ddl_output_mode=none
ADDITIONAL_OPTIONS+=('--no-left-queries-check')
ADDITIONAL_OPTIONS+=('--jobs')
ADDITIONAL_OPTIONS+=('2')
else

View File

@ -0,0 +1,20 @@
#!/bin/bash
set -e -x -a -u
ls -lha
cd hadoop-3.3.1
export JAVA_HOME=/usr
mkdir -p target/test/data
chown clickhouse ./target/test/data
sudo -E -u clickhouse bin/mapred minicluster -format -nomr -nnport 12222 &
while ! nc -z localhost 12222; do
sleep 1
done
lsof -i :12222
sleep 5

View File

@ -7,6 +7,10 @@
set -e -x -a -u
rpm2cpio ./minio-20220103182258.0.0.*.rpm | cpio -i --make-directories
find / -name minio
cp ./usr/local/bin/minio ./
ls -lha
mkdir -p ./minio_data
@ -25,12 +29,19 @@ fi
MINIO_ROOT_USER=${MINIO_ROOT_USER:-clickhouse}
MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-clickhouse}
./minio --version
./minio server --address ":11111" ./minio_data &
i=0
while ! curl -v --silent http://localhost:11111 2>&1 | grep AccessDenied
do
if [[ $i == 60 ]]; then
echo "Failed to setup minio"
exit 0
fi
echo "Trying to connect to minio"
sleep 1
i=$((i + 1))
done
lsof -i :11111

View File

@ -338,6 +338,7 @@ then
-e "Code: 1000, e.code() = 111, Connection refused" \
-e "UNFINISHED" \
-e "Renaming unexpected part" \
-e "PART_IS_TEMPORARILY_LOCKED" \
/var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv

View File

@ -6,8 +6,8 @@ Minimal ClickHouse build example:
```bash
cmake .. \
-DCMAKE_C_COMPILER=$(which clang-13) \
-DCMAKE_CXX_COMPILER=$(which clang++-13) \
-DCMAKE_C_COMPILER=$(which clang-14) \
-DCMAKE_CXX_COMPILER=$(which clang++-14) \
-DCMAKE_BUILD_TYPE=Debug \
-DENABLE_UTILS=OFF \
-DENABLE_TESTS=OFF

View File

@ -10,7 +10,7 @@ This is intended for continuous integration checks that run on Linux servers.
The cross-build for AARCH64 is based on the [Build instructions](../development/build.md), follow them first.
## Install Clang-13
## Install Clang-14 or newer
Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup or do
```
@ -31,7 +31,7 @@ tar xJf gcc-arm-8.3-2019.03-x86_64-aarch64-linux-gnu.tar.xz -C build-aarch64/cma
``` bash
cd ClickHouse
mkdir build-arm64
CC=clang-13 CXX=clang++-13 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-aarch64.cmake
CC=clang-14 CXX=clang++-14 cmake . -Bbuild-arm64 -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-aarch64.cmake
ninja -C build-arm64
```

View File

@ -10,14 +10,14 @@ This is intended for continuous integration checks that run on Linux servers. If
The cross-build for Mac OS X is based on the [Build instructions](../development/build.md), follow them first.
## Install Clang-13
## Install Clang-14
Follow the instructions from https://apt.llvm.org/ for your Ubuntu or Debian setup.
For example the commands for Bionic are like:
``` bash
sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-13 main" >> /etc/apt/sources.list
sudo apt-get install clang-13
sudo echo "deb [trusted=yes] http://apt.llvm.org/bionic/ llvm-toolchain-bionic-14 main" >> /etc/apt/sources.list
sudo apt-get install clang-14
```
## Install Cross-Compilation Toolset {#install-cross-compilation-toolset}

View File

@ -23,7 +23,7 @@ sudo bash -c "$(wget -O - https://apt.llvm.org/llvm.sh)"
``` bash
cd ClickHouse
mkdir build-riscv64
CC=clang-13 CXX=clang++-13 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_ORC=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF
CC=clang-14 CXX=clang++-14 cmake . -Bbuild-riscv64 -G Ninja -DCMAKE_TOOLCHAIN_FILE=cmake/linux/toolchain-riscv64.cmake -DGLIBC_COMPATIBILITY=OFF -DENABLE_LDAP=OFF -DOPENSSL_NO_ASM=ON -DENABLE_JEMALLOC=ON -DENABLE_PARQUET=OFF -DENABLE_ORC=OFF -DUSE_UNWIND=OFF -DENABLE_GRPC=OFF -DENABLE_HDFS=OFF -DENABLE_MYSQL=OFF
ninja -C build-riscv64
```

View File

@ -77,7 +77,7 @@ The build requires the following components:
- Git (is used only to checkout the sources, its not needed for the build)
- CMake 3.14 or newer
- Ninja
- C++ compiler: clang-13 or newer
- C++ compiler: clang-14 or newer
- Linker: lld
If all the components are installed, you may build in the same way as the steps above.

View File

@ -1,10 +1,10 @@
---
sidebar_position: 71
sidebar_label: Third-Party Libraries
description: A list of third-party libraries used
description: A list of third-party libraries used
---
# Third-Party Libraries Used
# Third-Party Libraries Used
The list of third-party libraries:

View File

@ -155,7 +155,7 @@ While inside the `build` directory, configure your build by running CMake. Befor
export CC=clang CXX=clang++
cmake ..
If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-13 CXX=clang++-13`. The clang version will be in the script output.
If you installed clang using the automatic installation script above, also specify the version of clang installed in the first command, e.g. `export CC=clang-14 CXX=clang++-14`. The clang version will be in the script output.
The `CC` variable specifies the compiler for C (short for C Compiler), and `CXX` variable instructs which C++ compiler is to be used for building.

View File

@ -3,6 +3,6 @@ sidebar_label: Development
sidebar_position: 58
---
# ClickHouse Development {#clickhouse-development}
# ClickHouse Development
[Original article](https://clickhouse.com/docs/en/development/) <!--hide-->

View File

@ -81,7 +81,7 @@ $ ./src/unit_tests_dbms --gtest_filter=LocalAddress*
## Performance Tests {#performance-tests}
Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/tests/performance-comparison` tool . See the readme file for invocation.
Performance tests allow to measure and compare performance of some isolated part of ClickHouse on synthetic queries. Tests are located at `tests/performance`. Each test is represented by `.xml` file with description of test case. Tests are run with `docker/test/performance-comparison` tool . See the readme file for invocation.
Each test run one or multiple queries (possibly with combinations of parameters) in a loop.

View File

@ -4,7 +4,7 @@ toc_priority: 27
toc_title: Introduction
---
# Database Engines {#database-engines}
# Database Engines
Database engines allow you to work with tables. By default, ClickHouse uses the [Atomic](../../engines/database-engines/atomic.md) database engine, which provides configurable [table engines](../../engines/table-engines/index.md) and an [SQL dialect](../../sql-reference/syntax.md).

View File

@ -3,7 +3,7 @@ sidebar_label: Lazy
sidebar_position: 20
---
# Lazy {#lazy}
# Lazy
Keeps tables in RAM only `expiration_time_in_seconds` seconds after last access. Can be used only with \*Log tables.

View File

@ -3,7 +3,7 @@ sidebar_label: MaterializedPostgreSQL
sidebar_position: 60
---
# [experimental] MaterializedPostgreSQL {#materialize-postgresql}
# [experimental] MaterializedPostgreSQL
Creates a ClickHouse database with tables from PostgreSQL database. Firstly, database with engine `MaterializedPostgreSQL` creates a snapshot of PostgreSQL database and loads required tables. Required tables can include any subset of tables from any subset of schemas from specified database. Along with the snapshot database engine acquires LSN and once initial dump of tables is performed - it starts pulling updates from WAL. After database is created, newly added tables to PostgreSQL database are not automatically added to replication. They have to be added manually with `ATTACH TABLE db.table` query.

View File

@ -3,7 +3,7 @@ sidebar_position: 40
sidebar_label: PostgreSQL
---
# PostgreSQL {#postgresql}
# PostgreSQL
Allows to connect to databases on a remote [PostgreSQL](https://www.postgresql.org) server. Supports read and write operations (`SELECT` and `INSERT` queries) to exchange data between ClickHouse and PostgreSQL.

View File

@ -3,7 +3,7 @@ sidebar_position: 30
sidebar_label: Replicated
---
# [experimental] Replicated {#replicated}
# [experimental] Replicated
The engine is based on the [Atomic](../../engines/database-engines/atomic.md) engine. It supports replication of metadata via DDL log being written to ZooKeeper and executed on all of the replicas for a given database.
@ -20,7 +20,7 @@ One ClickHouse server can have multiple replicated databases running and updatin
- `shard_name` — Shard name. Database replicas are grouped into shards by `shard_name`.
- `replica_name` — Replica name. Replica names must be different for all replicas of the same shard.
:::warning
:::warning
For [ReplicatedMergeTree](../table-engines/mergetree-family/replication.md#table_engines-replication) tables if no arguments provided, then default arguments are used: `/clickhouse/tables/{uuid}/{shard}` and `{replica}`. These can be changed in the server settings [default_replica_path](../../operations/server-configuration-parameters/settings.md#default_replica_path) and [default_replica_name](../../operations/server-configuration-parameters/settings.md#default_replica_name). Macro `{uuid}` is unfolded to table's uuid, `{shard}` and `{replica}` are unfolded to values from server config, not from database engine arguments. But in the future, it will be possible to use `shard_name` and `replica_name` of Replicated database.
:::

View File

@ -3,21 +3,21 @@ sidebar_position: 55
sidebar_label: SQLite
---
# SQLite {#sqlite}
# SQLite
Allows to connect to [SQLite](https://www.sqlite.org/index.html) database and perform `INSERT` and `SELECT` queries to exchange data between ClickHouse and SQLite.
Allows to connect to [SQLite](https://www.sqlite.org/index.html) database and perform `INSERT` and `SELECT` queries to exchange data between ClickHouse and SQLite.
## Creating a Database {#creating-a-database}
``` sql
CREATE DATABASE sqlite_database
CREATE DATABASE sqlite_database
ENGINE = SQLite('db_path')
```
**Engine Parameters**
- `db_path` — Path to a file with SQLite database.
## Data Types Support {#data_types-support}
| SQLite | ClickHouse |
@ -44,7 +44,7 @@ SHOW TABLES FROM sqlite_db;
``` text
┌──name───┐
│ table1 │
│ table2 │
│ table2 │
└─────────┘
```

View File

@ -4,7 +4,7 @@ toc_priority: 26
toc_title: Introduction
---
# Table Engines {#table_engines}
# Table Engines
The table engine (type of table) determines:

View File

@ -3,7 +3,7 @@ sidebar_position: 12
sidebar_label: ExternalDistributed
---
# ExternalDistributed {#externaldistributed}
# ExternalDistributed
The `ExternalDistributed` engine allows to perform `SELECT` queries on data that is stored on a remote servers MySQL or PostgreSQL. Accepts [MySQL](../../../engines/table-engines/integrations/mysql.md) or [PostgreSQL](../../../engines/table-engines/integrations/postgresql.md) engines as an argument so sharding is possible.

View File

@ -3,7 +3,7 @@ sidebar_position: 9
sidebar_label: EmbeddedRocksDB
---
# EmbeddedRocksDB Engine {#EmbeddedRocksDB-engine}
# EmbeddedRocksDB Engine
This engine allows integrating ClickHouse with [rocksdb](http://rocksdb.org/).

View File

@ -3,7 +3,7 @@ sidebar_position: 6
sidebar_label: HDFS
---
# HDFS {#table_engines-hdfs}
# HDFS
This engine provides integration with the [Apache Hadoop](https://en.wikipedia.org/wiki/Apache_Hadoop) ecosystem by allowing to manage data on [HDFS](https://hadoop.apache.org/docs/current/hadoop-project-dist/hadoop-hdfs/HdfsDesign.html) via ClickHouse. This engine is similar to the [File](../../../engines/table-engines/special/file.md#table_engines-file) and [URL](../../../engines/table-engines/special/url.md#table_engines-url) engines, but provides Hadoop-specific features.
@ -51,7 +51,7 @@ SELECT * FROM hdfs_engine_table LIMIT 2
## Implementation Details {#implementation-details}
- Reads and writes can be parallel.
- [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is supported.
- [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is supported.
- Not supported:
- `ALTER` and `SELECT...SAMPLE` operations.
- Indexes.
@ -98,7 +98,7 @@ Table consists of all the files in both directories (all files should satisfy fo
CREATE TABLE table_with_asterisk (name String, value UInt32) ENGINE = HDFS('hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV')
```
:::warning
:::warning
If the listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
:::

View File

@ -3,11 +3,11 @@ sidebar_position: 4
sidebar_label: Hive
---
# Hive {#hive}
# Hive
The Hive engine allows you to perform `SELECT` quries on HDFS Hive table. Currently it supports input formats as below:
- Text: only supports simple scalar column types except `binary`
- Text: only supports simple scalar column types except `binary`
- ORC: support simple scalar columns types except `char`; only support complex types like `array`
@ -66,33 +66,33 @@ When ClickHouse is started up with local cache for remote filesystem enabled, us
#### Create Table in Hive
``` text
hive > CREATE TABLE `test`.`test_orc`(
`f_tinyint` tinyint,
`f_smallint` smallint,
`f_int` int,
`f_integer` int,
`f_bigint` bigint,
`f_float` float,
`f_double` double,
`f_decimal` decimal(10,0),
`f_timestamp` timestamp,
`f_date` date,
`f_string` string,
`f_varchar` varchar(100),
`f_bool` boolean,
`f_binary` binary,
`f_array_int` array<int>,
`f_array_string` array<string>,
`f_array_float` array<float>,
`f_array_array_int` array<array<int>>,
`f_array_array_string` array<array<string>>,
`f_tinyint` tinyint,
`f_smallint` smallint,
`f_int` int,
`f_integer` int,
`f_bigint` bigint,
`f_float` float,
`f_double` double,
`f_decimal` decimal(10,0),
`f_timestamp` timestamp,
`f_date` date,
`f_string` string,
`f_varchar` varchar(100),
`f_bool` boolean,
`f_binary` binary,
`f_array_int` array<int>,
`f_array_string` array<string>,
`f_array_float` array<float>,
`f_array_array_int` array<array<int>>,
`f_array_array_string` array<array<string>>,
`f_array_array_float` array<array<float>>)
PARTITIONED BY (
PARTITIONED BY (
`day` string)
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
OUTPUTFORMAT
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.orc.OrcSerde'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.orc.OrcInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat'
LOCATION
'hdfs://testcluster/data/hive/test.db/test_orc'
@ -178,7 +178,7 @@ f_array_array_float: [[1.11,2.22],[3.33,4.44]]
day: 2021-09-18
1 rows in set. Elapsed: 0.078 sec.
1 rows in set. Elapsed: 0.078 sec.
```
### Query Hive Table with Parquet Input Format
@ -187,34 +187,34 @@ day: 2021-09-18
``` text
hive >
CREATE TABLE `test`.`test_parquet`(
`f_tinyint` tinyint,
`f_smallint` smallint,
`f_int` int,
`f_integer` int,
`f_bigint` bigint,
`f_float` float,
`f_double` double,
`f_decimal` decimal(10,0),
`f_timestamp` timestamp,
`f_date` date,
`f_string` string,
`f_varchar` varchar(100),
`f_char` char(100),
`f_bool` boolean,
`f_binary` binary,
`f_array_int` array<int>,
`f_array_string` array<string>,
`f_array_float` array<float>,
`f_array_array_int` array<array<int>>,
`f_array_array_string` array<array<string>>,
`f_tinyint` tinyint,
`f_smallint` smallint,
`f_int` int,
`f_integer` int,
`f_bigint` bigint,
`f_float` float,
`f_double` double,
`f_decimal` decimal(10,0),
`f_timestamp` timestamp,
`f_date` date,
`f_string` string,
`f_varchar` varchar(100),
`f_char` char(100),
`f_bool` boolean,
`f_binary` binary,
`f_array_int` array<int>,
`f_array_string` array<string>,
`f_array_float` array<float>,
`f_array_array_int` array<array<int>>,
`f_array_array_string` array<array<string>>,
`f_array_array_float` array<array<float>>)
PARTITIONED BY (
PARTITIONED BY (
`day` string)
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
ROW FORMAT SERDE
'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
LOCATION
'hdfs://testcluster/data/hive/test.db/test_parquet'
@ -299,7 +299,7 @@ f_array_array_string: [['a','b'],['c','d']]
f_array_array_float: [[1.11,2.22],[3.33,4.44]]
day: 2021-09-18
1 rows in set. Elapsed: 0.357 sec.
1 rows in set. Elapsed: 0.357 sec.
```
### Query Hive Table with Text Input Format
@ -307,34 +307,34 @@ day: 2021-09-18
``` text
hive >
CREATE TABLE `test`.`test_text`(
`f_tinyint` tinyint,
`f_smallint` smallint,
`f_int` int,
`f_integer` int,
`f_bigint` bigint,
`f_float` float,
`f_double` double,
`f_decimal` decimal(10,0),
`f_timestamp` timestamp,
`f_date` date,
`f_string` string,
`f_varchar` varchar(100),
`f_char` char(100),
`f_bool` boolean,
`f_binary` binary,
`f_array_int` array<int>,
`f_array_string` array<string>,
`f_array_float` array<float>,
`f_array_array_int` array<array<int>>,
`f_array_array_string` array<array<string>>,
`f_tinyint` tinyint,
`f_smallint` smallint,
`f_int` int,
`f_integer` int,
`f_bigint` bigint,
`f_float` float,
`f_double` double,
`f_decimal` decimal(10,0),
`f_timestamp` timestamp,
`f_date` date,
`f_string` string,
`f_varchar` varchar(100),
`f_char` char(100),
`f_bool` boolean,
`f_binary` binary,
`f_array_int` array<int>,
`f_array_string` array<string>,
`f_array_float` array<float>,
`f_array_array_int` array<array<int>>,
`f_array_array_string` array<array<string>>,
`f_array_array_float` array<array<float>>)
PARTITIONED BY (
PARTITIONED BY (
`day` string)
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe'
STORED AS INPUTFORMAT
'org.apache.hadoop.mapred.TextInputFormat'
OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat'
LOCATION
'hdfs://testcluster/data/hive/test.db/test_text'
@ -374,7 +374,7 @@ CREATE TABLE test.test_text
`day` String
)
ENGINE = Hive('thrift://localhost:9083', 'test', 'test_text')
PARTITION BY day
PARTITION BY day
```
``` sql

View File

@ -3,7 +3,7 @@ sidebar_position: 40
sidebar_label: Integrations
---
# Table Engines for Integrations {#table-engines-for-integrations}
# Table Engines for Integrations
ClickHouse provides various means for integrating with external systems, including table engines. Like with all other table engines, the configuration is done using `CREATE TABLE` or `ALTER TABLE` queries. Then from a user perspective, the configured integration looks like a normal table, but queries to it are proxied to the external system. This transparent querying is one of the key advantages of this approach over alternative integration methods, like external dictionaries or table functions, which require to use custom query methods on each use.

View File

@ -3,7 +3,7 @@ sidebar_position: 3
sidebar_label: JDBC
---
# JDBC {#table-engine-jdbc}
# JDBC
Allows ClickHouse to connect to external databases via [JDBC](https://en.wikipedia.org/wiki/Java_Database_Connectivity).

View File

@ -3,7 +3,7 @@ sidebar_position: 8
sidebar_label: Kafka
---
# Kafka {#kafka}
# Kafka
This engine works with [Apache Kafka](http://kafka.apache.org/).
@ -87,7 +87,7 @@ Examples:
<summary>Deprecated Method for Creating a Table</summary>
:::warning
:::warning
Do not use this method in new projects. If possible, switch old projects to the method described above.
:::

View File

@ -3,7 +3,7 @@ sidebar_position: 12
sidebar_label: MaterializedPostgreSQL
---
# MaterializedPostgreSQL {#materialize-postgresql}
# MaterializedPostgreSQL
Creates ClickHouse table with an initial data dump of PostgreSQL table and starts replication process, i.e. executes background job to apply new changes as they happen on PostgreSQL table in the remote PostgreSQL database.
@ -52,7 +52,7 @@ PRIMARY KEY key;
SELECT key, value, _version FROM postgresql_db.postgresql_replica;
```
:::warning
:::warning
Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.html) values is not supported. The default value for the data type will be used.
:::

View File

@ -3,7 +3,7 @@ sidebar_position: 5
sidebar_label: MongoDB
---
# MongoDB {#mongodb}
# MongoDB
MongoDB engine is read-only table engine which allows to read data (`SELECT` queries) from remote MongoDB collection. Engine supports only non-nested data types. `INSERT` queries are not supported.

View File

@ -3,7 +3,7 @@ sidebar_position: 4
sidebar_label: MySQL
---
# MySQL {#mysql}
# MySQL
The MySQL engine allows you to perform `SELECT` and `INSERT` queries on data that is stored on a remote MySQL server.

View File

@ -3,7 +3,7 @@ sidebar_position: 2
sidebar_label: ODBC
---
# ODBC {#table-engine-odbc}
# ODBC
Allows ClickHouse to connect to external databases via [ODBC](https://en.wikipedia.org/wiki/Open_Database_Connectivity).

View File

@ -3,7 +3,7 @@ sidebar_position: 11
sidebar_label: PostgreSQL
---
# PostgreSQL {#postgresql}
# PostgreSQL
The PostgreSQL engine allows to perform `SELECT` and `INSERT` queries on data that is stored on a remote PostgreSQL server.

View File

@ -3,7 +3,7 @@ sidebar_position: 10
sidebar_label: RabbitMQ
---
# RabbitMQ Engine {#rabbitmq-engine}
# RabbitMQ Engine
This engine allows integrating ClickHouse with [RabbitMQ](https://www.rabbitmq.com).

View File

@ -3,7 +3,7 @@ sidebar_position: 7
sidebar_label: S3
---
# S3 Table Engine {#table-engine-s3}
# S3 Table Engine
This engine provides integration with [Amazon S3](https://aws.amazon.com/s3/) ecosystem. This engine is similar to the [HDFS](../../../engines/table-engines/special/file.md#table_engines-hdfs) engine, but provides S3-specific features.
@ -25,7 +25,7 @@ CREATE TABLE s3_engine_table (name String, value UInt32)
**Example**
``` sql
CREATE TABLE s3_engine_table (name String, value UInt32)
CREATE TABLE s3_engine_table (name String, value UInt32)
ENGINE=S3('https://clickhouse-public-datasets.s3.amazonaws.com/my-test-bucket-768/test-data.csv.gz', 'CSV', 'gzip')
SETTINGS input_format_with_names_use_header = 0;
@ -50,7 +50,7 @@ For more information about virtual columns see [here](../../../engines/table-eng
## Implementation Details {#implementation-details}
- Reads and writes can be parallel
- [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is supported.
- [Zero-copy](../../../operations/storing-data.md#zero-copy) replication is supported.
- Not supported:
- `ALTER` and `SELECT...SAMPLE` operations.
- Indexes.
@ -66,7 +66,7 @@ For more information about virtual columns see [here](../../../engines/table-eng
Constructions with `{}` are similar to the [remote](../../../sql-reference/table-functions/remote.md) table function.
:::warning
:::warning
If the listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
:::

View File

@ -3,16 +3,16 @@ sidebar_position: 7
sidebar_label: SQLite
---
# SQLite {#sqlite}
# SQLite
The engine allows to import and export data to SQLite and supports queries to SQLite tables directly from ClickHouse.
## Creating a Table {#creating-a-table}
``` sql
CREATE TABLE [IF NOT EXISTS] [db.]table_name
CREATE TABLE [IF NOT EXISTS] [db.]table_name
(
name1 [type1],
name1 [type1],
name2 [type2], ...
) ENGINE = SQLite('db_path', 'table')
```
@ -32,10 +32,10 @@ SHOW CREATE TABLE sqlite_db.table2;
``` text
CREATE TABLE SQLite.table2
(
`col1` Nullable(Int32),
(
`col1` Nullable(Int32),
`col2` Nullable(String)
)
)
ENGINE = SQLite('sqlite.db','table2');
```

View File

@ -3,7 +3,7 @@ sidebar_position: 20
sidebar_label: Log Family
---
# Log Engine Family {#log-engine-family}
# Log Engine Family
These engines were developed for scenarios when you need to quickly write many small tables (up to about 1 million rows) and read them later as a whole.

View File

@ -3,7 +3,7 @@ toc_priority: 33
toc_title: Log
---
# Log {#log}
# Log
The engine belongs to the family of `Log` engines. See the common properties of `Log` engines and their differences in the [Log Engine Family](../../../engines/table-engines/log-family/index.md) article.

View File

@ -3,7 +3,7 @@ toc_priority: 32
toc_title: StripeLog
---
# Stripelog {#stripelog}
# Stripelog
This engine belongs to the family of log engines. See the common properties of log engines and their differences in the [Log Engine Family](../../../engines/table-engines/log-family/index.md) article.

View File

@ -3,7 +3,7 @@ toc_priority: 34
toc_title: TinyLog
---
# TinyLog {#tinylog}
# TinyLog
The engine belongs to the log engine family. See [Log Engine Family](../../../engines/table-engines/log-family/index.md) for common properties of log engines and their differences.

View File

@ -3,7 +3,7 @@ sidebar_position: 60
sidebar_label: AggregatingMergeTree
---
# AggregatingMergeTree {#aggregatingmergetree}
# AggregatingMergeTree
The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree), altering the logic for data parts merging. ClickHouse replaces all rows with the same primary key (or more accurately, with the same [sorting key](../../../engines/table-engines/mergetree-family/mergetree.md)) with a single row (within a one data part) that stores a combination of states of aggregate functions.

View File

@ -3,7 +3,7 @@ sidebar_position: 70
sidebar_label: CollapsingMergeTree
---
# CollapsingMergeTree {#table_engine-collapsingmergetree}
# CollapsingMergeTree
The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md) and adds the logic of rows collapsing to data parts merge algorithm.
@ -42,7 +42,7 @@ When creating a `CollapsingMergeTree` table, the same [query clauses](../../../e
<summary>Deprecated Method for Creating a Table</summary>
:::warning
:::warning
Do not use this method in new projects and, if possible, switch old projects to the method described above.
:::

View File

@ -3,10 +3,10 @@ sidebar_position: 30
sidebar_label: Custom Partitioning Key
---
# Custom Partitioning Key {#custom-partitioning-key}
# Custom Partitioning Key
:::warning
In most cases you do not need a partition key, and in most other cases you do not need a partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression).
:::warning
In most cases you do not need a partition key, and in most other cases you do not need a partition key more granular than by months. Partitioning does not speed up queries (in contrast to the ORDER BY expression).
You should never use too granular of partitioning. Don't partition your data by client identifiers or names. Instead, make a client identifier or name the first column in the ORDER BY expression.
:::
@ -43,7 +43,7 @@ By default, the floating-point partition key is not supported. To use it enable
When inserting new data to a table, this data is stored as a separate part (chunk) sorted by the primary key. In 10-15 minutes after inserting, the parts of the same partition are merged into the entire part.
:::info
:::info
A merge only works for data parts that have the same value for the partitioning expression. This means **you shouldnt make overly granular partitions** (more than about a thousand partitions). Otherwise, the `SELECT` query performs poorly because of an unreasonably large number of files in the file system and open file descriptors.
:::

View File

@ -3,7 +3,7 @@ sidebar_position: 90
sidebar_label: GraphiteMergeTree
---
# GraphiteMergeTree {#graphitemergetree}
# GraphiteMergeTree
This engine is designed for thinning and aggregating/averaging (rollup) [Graphite](http://graphite.readthedocs.io/en/latest/index.html) data. It may be helpful to developers who want to use ClickHouse as a data store for Graphite.
@ -54,7 +54,7 @@ When creating a `GraphiteMergeTree` table, the same [clauses](../../../engines/t
<summary>Deprecated Method for Creating a Table</summary>
:::warning
:::warning
Do not use this method in new projects and, if possible, switch old projects to the method described above.
:::
@ -120,7 +120,7 @@ default
...
```
:::warning
:::warning
Patterns must be strictly ordered:
1. Patterns without `function` or `retention`.
@ -132,9 +132,9 @@ When processing a row, ClickHouse checks the rules in the `pattern` sections. Ea
Fields for `pattern` and `default` sections:
- `rule_type` - a rule's type. It's applied only to a particular metrics. The engine use it to separate plain and tagged metrics. Optional parameter. Default value: `all`.
It's unnecessary when performance is not critical, or only one metrics type is used, e.g. plain metrics. By default only one type of rules set is created. Otherwise, if any of special types is defined, two different sets are created. One for plain metrics (root.branch.leaf) and one for tagged metrics (root.branch.leaf;tag1=value1).
The default rules are ended up in both sets.
- `rule_type` - a rule's type. It's applied only to a particular metrics. The engine use it to separate plain and tagged metrics. Optional parameter. Default value: `all`.
It's unnecessary when performance is not critical, or only one metrics type is used, e.g. plain metrics. By default only one type of rules set is created. Otherwise, if any of special types is defined, two different sets are created. One for plain metrics (root.branch.leaf) and one for tagged metrics (root.branch.leaf;tag1=value1).
The default rules are ended up in both sets.
Valid values:
- `all` (default) - a universal rule, used when `rule_type` is omitted.
- `plain` - a rule for plain metrics. The field `regexp` is processed as regular expression.
@ -143,7 +143,7 @@ Valid values:
- `regexp` A pattern for the metric name (a regular or DSL).
- `age` The minimum age of the data in seconds.
- `precision` How precisely to define the age of the data in seconds. Should be a divisor for 86400 (seconds in a day).
- `function` The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. Accepted functions: min / max / any / avg. The average is calculated imprecisely, like the average of the averages.
- `function` The name of the aggregating function to apply to data whose age falls within the range `[age, age + precision]`. Accepted functions: min / max / any / avg. The average is calculated imprecisely, like the average of the averages.
### Configuration Example without rules types {#configuration-example}

View File

@ -3,7 +3,7 @@ sidebar_position: 10
sidebar_label: MergeTree Family
---
# MergeTree Engine Family {#mergetree-engine-family}
# MergeTree Engine Family
Table engines from the MergeTree family are the core of ClickHouse data storage capabilities. They provide most features for resilience and high-performance data retrieval: columnar storage, custom partitioning, sparse primary index, secondary data-skipping indexes, etc.

View File

@ -3,7 +3,7 @@ sidebar_position: 11
sidebar_label: MergeTree
---
# MergeTree {#table_engines-mergetree}
# MergeTree
The `MergeTree` engine and other engines of this family (`*MergeTree`) are the most robust ClickHouse table engines.
@ -694,7 +694,7 @@ Tags:
- `volume_name_N` — Volume name. Volume names must be unique.
- `disk` — a disk within a volume.
- `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volumes disks. If the a size of a merged part estimated to be bigger than `max_data_part_size_bytes` then this part will be written to a next volume. Basically this feature allows to keep new/small parts on a hot (SSD) volume and move them to a cold (HDD) volume when they reach large size. Do not use this setting if your policy has only one volume.
- `move_factor` — when the amount of available space gets lower than this factor, data automatically starts to move on the next volume if any (by default, 0.1). ClickHouse sorts existing parts by size from largest to smallest (in descending order) and selects parts with the total size that is sufficient to meet the `move_factor` condition. If the total size of all parts is insufficient, all parts will be moved.
- `move_factor` — when the amount of available space gets lower than this factor, data automatically starts to move on the next volume if any (by default, 0.1). ClickHouse sorts existing parts by size from largest to smallest (in descending order) and selects parts with the total size that is sufficient to meet the `move_factor` condition. If the total size of all parts is insufficient, all parts will be moved.
- `prefer_not_to_merge` — Disables merging of data parts on this volume. When this setting is enabled, merging data on this volume is not allowed. This allows controlling how ClickHouse works with slow disks.
- `perform_ttl_move_on_insert` — Disables TTL move on data part INSERT. By default if we insert a data part that already expired by the TTL move rule it immediately goes to a volume/disk declared in move rule. This can significantly slowdown insert in case if destination volume/disk is slow (e.g. S3).
- `load_balancing` - Policy for disk balancing, `round_robin` or `least_used`.

View File

@ -3,7 +3,7 @@ sidebar_position: 40
sidebar_label: ReplacingMergeTree
---
# ReplacingMergeTree {#replacingmergetree}
# ReplacingMergeTree
The engine differs from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree) in that it removes duplicate entries with the same [sorting key](../../../engines/table-engines/mergetree-family/mergetree.md) value (`ORDER BY` table section, not `PRIMARY KEY`).

View File

@ -3,7 +3,7 @@ sidebar_position: 20
sidebar_label: Data Replication
---
# Data Replication {#table_engines-replication}
# Data Replication
Replication is only supported for tables in the MergeTree family:
@ -112,7 +112,7 @@ Data blocks are deduplicated. For multiple writes of the same data block (data b
During replication, only the source data to insert is transferred over the network. Further data transformation (merging) is coordinated and performed on all the replicas in the same way. This minimizes network usage, which means that replication works well when replicas reside in different datacenters. (Note that duplicating data in different datacenters is the main goal of replication.)
You can have any number of replicas of the same data. Based on our experiences, a relatively reliable and convenient solution could use double replication in production, with each server using RAID-5 or RAID-6 (and RAID-10 in some cases).
You can have any number of replicas of the same data. Based on our experiences, a relatively reliable and convenient solution could use double replication in production, with each server using RAID-5 or RAID-6 (and RAID-10 in some cases).
The system monitors data synchronicity on replicas and is able to recover after a failure. Failover is automatic (for small differences in data) or semi-automatic (when data differs too much, which may indicate a configuration error).

View File

@ -3,7 +3,7 @@ sidebar_position: 50
sidebar_label: SummingMergeTree
---
# SummingMergeTree {#summingmergetree}
# SummingMergeTree
The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-family/mergetree.md#table_engines-mergetree). The difference is that when merging data parts for `SummingMergeTree` tables ClickHouse replaces all the rows with the same primary key (or more accurately, with the same [sorting key](../../../engines/table-engines/mergetree-family/mergetree.md)) with one row which contains summarized values for the columns with the numeric data type. If the sorting key is composed in a way that a single key value corresponds to large number of rows, this significantly reduces storage volume and speeds up data selection.

View File

@ -3,7 +3,7 @@ sidebar_position: 80
sidebar_label: VersionedCollapsingMergeTree
---
# VersionedCollapsingMergeTree {#versionedcollapsingmergetree}
# VersionedCollapsingMergeTree
This engine:
@ -53,7 +53,7 @@ When creating a `VersionedCollapsingMergeTree` table, the same [clauses](../../.
<summary>Deprecated Method for Creating a Table</summary>
:::warning
:::warning
Do not use this method in new projects. If possible, switch old projects to the method described above.
:::

View File

@ -3,7 +3,7 @@ sidebar_position: 120
sidebar_label: Buffer
---
# Buffer Table Engine {#buffer}
# Buffer Table Engine
Buffers the data to write in RAM, periodically flushing it to another table. During the read operation, data is read from the buffer and the other table simultaneously.
@ -54,7 +54,7 @@ If the set of columns in the Buffer table does not match the set of columns in a
If the types do not match for one of the columns in the Buffer table and a subordinate table, an error message is entered in the server log, and the buffer is cleared.
The same thing happens if the subordinate table does not exist when the buffer is flushed.
:::warning
:::warning
Running ALTER on the Buffer table in releases made before 26 Oct 2021 will cause a `Block structure mismatch` error (see [#15117](https://github.com/ClickHouse/ClickHouse/issues/15117) and [#30565](https://github.com/ClickHouse/ClickHouse/pull/30565)), so deleting the Buffer table and then recreating is the only option. It is advisable to check that this error is fixed in your release before trying to run ALTER on the Buffer table.
:::

View File

@ -3,7 +3,7 @@ sidebar_position: 20
sidebar_label: Dictionary
---
# Dictionary Table Engine {#dictionary}
# Dictionary Table Engine
The `Dictionary` engine displays the [dictionary](../../../sql-reference/dictionaries/external-dictionaries/external-dicts.md) data as a ClickHouse table.

Some files were not shown because too many files have changed in this diff Show More