thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

merge master

Browse Source
This commit is contained in:
taiyang-li 2021-12-27 15:12:48 +08:00
commit 9036b18c2f
388 changed files with 5853 additions and 3543 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/backport_branches.yml vendored
View File

@ -95,6 +95,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -139,6 +140,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -183,6 +185,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -227,6 +230,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}

44
.github/workflows/jepsen.yml vendored Normal file
View File

@ -0,0 +1,44 @@
name: JepsenWorkflow
env:
# Force the stdout and stderr streams to be unbuffered
PYTHONUNBUFFERED: 1
concurrency:
group: jepsen
on: # yamllint disable-line rule:truthy
schedule:
- cron: '0 */6 * * *'
workflow_run:
workflows: ["CIGithubActions"]
types:
- completed
workflow_dispatch:
jobs:
KeeperJepsenRelease:
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/keeper_jepsen
REPO_COPY=${{runner.temp}}/keeper_jepsen/ClickHouse
EOF
- name: Clear repository
run: |
sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
- name: Check out repository code
uses: actions/checkout@v2
with:
fetch-depth: 0
- name: Jepsen Test
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci
python3 keeper_jepsen_check.py
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH

20
.github/workflows/main.yml vendored
View File

@ -236,6 +236,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -280,6 +281,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -324,6 +326,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -365,6 +368,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -409,6 +413,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -453,6 +458,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -497,6 +503,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -541,6 +548,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -585,6 +593,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -632,6 +641,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -676,6 +686,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -720,6 +731,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -764,6 +776,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -808,6 +821,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -852,6 +866,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -896,6 +911,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -920,7 +936,7 @@ jobs:
- BuilderDebMsan
- BuilderDebDebug
runs-on: [self-hosted, style-checker]
if: always()
if: ${{ success() || failure() }}
steps:
- name: Set envs
run: |
@ -960,7 +976,7 @@ jobs:
- BuilderBinDarwinAarch64
- BuilderBinPPC64
runs-on: [self-hosted, style-checker]
if: always()
if: ${{ success() || failure() }}
steps:
- name: Set envs
run: |

15
.github/workflows/master.yml vendored
View File

@ -157,6 +157,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -201,6 +202,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -246,6 +248,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -290,6 +293,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -334,6 +338,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -378,6 +383,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -422,6 +428,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -466,6 +473,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -514,6 +522,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -559,6 +568,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -604,6 +614,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -649,6 +660,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -694,6 +706,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -739,6 +752,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -784,6 +798,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}

6
.github/workflows/release_branches.yml vendored
View File

@ -98,6 +98,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -142,6 +143,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -186,6 +188,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -230,6 +233,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -274,6 +278,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
@ -318,6 +323,7 @@ jobs:
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 build_check.py "$CHECK_NAME" $BUILD_NAME
- name: Upload build URLs to artifacts
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}

42
.github/workflows/woboq.yml vendored Normal file
View File

@ -0,0 +1,42 @@
name: WoboqBuilder
env:
# Force the stdout and stderr streams to be unbuffered
PYTHONUNBUFFERED: 1
concurrency:
group: woboq
on: # yamllint disable-line rule:truthy
schedule:
- cron: '0 */18 * * *'
workflow_dispatch:
jobs:
# don't use dockerhub push because this image updates so rarely
WoboqCodebrowser:
runs-on: [self-hosted, style-checker]
steps:
- name: Set envs
run: |
cat >> "$GITHUB_ENV" << 'EOF'
TEMP_PATH=${{runner.temp}}/codebrowser
REPO_COPY=${{runner.temp}}/codebrowser/ClickHouse
IMAGES_PATH=${{runner.temp}}/images_path
EOF
- name: Clear repository
run: |
sudo rm -fr $GITHUB_WORKSPACE && mkdir $GITHUB_WORKSPACE
- name: Check out repository code
uses: actions/checkout@v2
with:
submodules: 'true'
- name: Codebrowser
run: |
sudo rm -fr $TEMP_PATH
mkdir -p $TEMP_PATH
cp -r $GITHUB_WORKSPACE $TEMP_PATH
cd $REPO_COPY/tests/ci && python3 codebrowser_check.py
- name: Cleanup
if: always()
run: |
docker kill $(docker ps -q) ||:
docker rm -f $(docker ps -a -q) ||:
sudo rm -fr $TEMP_PATH

8
.gitmodules vendored
View File

@ -54,8 +54,8 @@
url = https://github.com/ClickHouse-Extras/Turbo-Base64.git
[submodule "contrib/arrow"]
path = contrib/arrow
url = https://github.com/ClickHouse-Extras/arrow
branch = clickhouse-arrow-2.0.0
url = https://github.com/ClickHouse-Extras/arrow.git
branch = blessed/release-6.0.1
[submodule "contrib/thrift"]
path = contrib/thrift
url = https://github.com/apache/thrift.git
@ -190,8 +190,8 @@
url = https://github.com/xz-mirror/xz
[submodule "contrib/abseil-cpp"]
path = contrib/abseil-cpp
url = https://github.com/ClickHouse-Extras/abseil-cpp.git
branch = lts_2020_02_25
url = https://github.com/abseil/abseil-cpp.git
branch = lts_2021_11_02
[submodule "contrib/dragonbox"]
path = contrib/dragonbox
url = https://github.com/ClickHouse-Extras/dragonbox.git

10
CMakeLists.txt
View File

@ -424,6 +424,11 @@ if (OS_LINUX AND NOT SANITIZE)
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--no-undefined")
endif ()
# Increase stack size on Musl. We need big stack for our recursive-descend parser.
if (USE_MUSL)
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,-z,stack-size=2097152")
endif ()
include(cmake/dbms_glob_sources.cmake)
if (OS_LINUX OR OS_ANDROID)
@ -451,6 +456,11 @@ if (MAKE_STATIC_LIBRARIES)
endif ()
else ()
set (CMAKE_POSITION_INDEPENDENT_CODE ON)
# This is required for clang on Arch linux, that uses PIE by default.
# See enable-SSP-and-PIE-by-default.patch [1].
#
# [1]: https://github.com/archlinux/svntogit-packages/blob/6e681aa860e65ad46a1387081482eb875c2200f2/trunk/enable-SSP-and-PIE-by-default.patch
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -no-pie")
endif ()
if (ENABLE_TESTS)

3
PreLoad.cmake
View File

@ -27,8 +27,7 @@ execute_process(COMMAND uname -m OUTPUT_VARIABLE ARCH)
if (OS MATCHES "Linux"
AND NOT DEFINED CMAKE_TOOLCHAIN_FILE
AND NOT DISABLE_HERMETIC_BUILD
AND ($ENV{CC} MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*")
AND (USE_STATIC_LIBRARIES OR NOT DEFINED USE_STATIC_LIBRARIES))
AND ($ENV{CC} MATCHES ".*clang.*" OR CMAKE_C_COMPILER MATCHES ".*clang.*"))
if (ARCH MATCHES "amd64|x86_64")
set (CMAKE_TOOLCHAIN_FILE "cmake/linux/toolchain-x86_64.cmake" CACHE INTERNAL "" FORCE)

4
base/CMakeLists.txt
View File

@ -9,7 +9,3 @@ add_subdirectory (pcg-random)
add_subdirectory (widechar_width)
add_subdirectory (readpassphrase)
add_subdirectory (bridge)
if (USE_MYSQL)
add_subdirectory (mysqlxx)
endif ()

3
base/base/CMakeLists.txt
View File

@ -1,8 +1,6 @@
set (SRCS
argsToConfig.cpp
coverage.cpp
DateLUT.cpp
DateLUTImpl.cpp
demangle.cpp
getFQDNOrHostName.cpp
getMemoryAmount.cpp
@ -18,7 +16,6 @@ set (SRCS
sleep.cpp
terminalColors.cpp
errnoToString.cpp
getResource.cpp
StringRef.cpp
)

7
base/base/getPageSize.cpp
View File

@ -1,8 +1,11 @@
#include <base/getPageSize.h>
#include <unistd.h>
#include <cstdlib>
Int64 getPageSize()
{
return sysconf(_SC_PAGESIZE);
Int64 page_size = sysconf(_SC_PAGESIZE);
if (page_size < 0)
abort();
return page_size;
}

8
base/base/phdr_cache.cpp
View File

@ -123,6 +123,12 @@ bool hasPHDRCache()
#else
void updatePHDRCache() {}
bool hasPHDRCache() { return false; }
#if defined(USE_MUSL)
/// With statically linked with musl, dl_iterate_phdr is immutable.
bool hasPHDRCache() { return true; }
#else
bool hasPHDRCache() { return false; }
#endif
#endif

23
base/harmful/harmful.c
View File

@ -182,7 +182,6 @@ TRAP(vlimit)
TRAP(wcsnrtombs)
TRAP(wcsrtombs)
TRAP(wctomb)
TRAP(wordexp)
TRAP(basename)
TRAP(catgets)
TRAP(dbm_clearerr)
@ -195,9 +194,8 @@ TRAP(dbm_nextkey)
TRAP(dbm_open)
TRAP(dbm_store)
TRAP(dirname)
#if !defined(SANITIZER)
TRAP(dlerror) // Used by tsan
#endif
// TRAP(dlerror) // It is not thread-safe. But it is used by dynamic linker to load some name resolution plugins. Also used by TSan.
/// Note: we should better get rid of glibc, dynamic linking and all that sort of annoying garbage altogether.
TRAP(ftw)
TRAP(getc_unlocked)
//TRAP(getenv) // Ok at program startup
@ -245,4 +243,21 @@ TRAP(lgammaf32x)
TRAP(lgammaf64)
TRAP(lgammaf64x)
/// These functions are unused by ClickHouse and we should be aware if they are accidentally get used.
/// Sometimes people report that these function contain vulnerabilities (these reports are bogus for ClickHouse).
TRAP(mq_close)
TRAP(mq_getattr)
TRAP(mq_setattr)
TRAP(mq_notify)
TRAP(mq_open)
TRAP(mq_receive)
TRAP(mq_send)
TRAP(mq_unlink)
TRAP(mq_timedsend)
TRAP(mq_timedreceive)
/// These functions are also unused by ClickHouse.
TRAP(wordexp)
TRAP(wordfree)
#endif

61
base/mysqlxx/CMakeLists.txt
View File

@ -1,61 +0,0 @@
add_library (mysqlxx
Connection.cpp
Exception.cpp
Query.cpp
ResultBase.cpp
UseQueryResult.cpp
Row.cpp
Value.cpp
Pool.cpp
PoolFactory.cpp
PoolWithFailover.cpp
)
target_include_directories (mysqlxx PUBLIC ..)
if (NOT USE_INTERNAL_MYSQL_LIBRARY)
set(PLATFORM_LIBRARIES ${CMAKE_DL_LIBS})
if (USE_MYSQL)
target_include_directories (mysqlxx SYSTEM PRIVATE ${MYSQL_INCLUDE_DIR})
endif ()
if (APPLE)
find_library (ICONV_LIBRARY iconv)
set (MYSQLCLIENT_LIBRARIES ${MYSQLCLIENT_LIBRARIES} ${STATIC_MYSQLCLIENT_LIB} ${ICONV_LIBRARY})
elseif (USE_STATIC_LIBRARIES AND STATIC_MYSQLCLIENT_LIB)
set (MYSQLCLIENT_LIBRARIES ${STATIC_MYSQLCLIENT_LIB})
endif ()
endif ()
target_link_libraries (mysqlxx
PUBLIC
common
PRIVATE
${MYSQLCLIENT_LIBRARIES}
${ZLIB_LIBRARIES}
)
if(OPENSSL_LIBRARIES)
target_link_libraries(mysqlxx PRIVATE ${OPENSSL_LIBRARIES})
endif()
target_link_libraries(mysqlxx PRIVATE ${PLATFORM_LIBRARIES})
if (NOT USE_INTERNAL_MYSQL_LIBRARY AND OPENSSL_INCLUDE_DIR)
target_include_directories (mysqlxx SYSTEM PRIVATE ${OPENSSL_INCLUDE_DIR})
endif ()
target_no_warning(mysqlxx reserved-macro-identifier)
if (NOT USE_INTERNAL_MYSQL_LIBRARY AND USE_STATIC_LIBRARIES)
message(WARNING "Statically linking with system mysql/mariadb only works "
"if mysql client libraries are built with same openssl version as "
"we are going to use now. It wouldn't work if GnuTLS is used. "
"Try -D\"USE_INTERNAL_MYSQL_LIBRARY\"=ON or -D\"ENABLE_MYSQL\"=OFF or "
"-D\"USE_STATIC_LIBRARIES\"=OFF")
endif ()
if (ENABLE_TESTS)
add_subdirectory (tests)
endif ()

12
cmake/find/ccache.cmake
View File

@ -32,11 +32,6 @@ if (CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE)
if (CCACHE_VERSION VERSION_GREATER "3.2.0" OR NOT CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
message(STATUS "Using ${CCACHE_FOUND} ${CCACHE_VERSION}")
set (CMAKE_CXX_COMPILER_LAUNCHER ${CCACHE_FOUND} ${CMAKE_CXX_COMPILER_LAUNCHER})
set (CMAKE_C_COMPILER_LAUNCHER ${CCACHE_FOUND} ${CMAKE_C_COMPILER_LAUNCHER})
set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
# debian (debhelpers) set SOURCE_DATE_EPOCH environment variable, that is
# filled from the debian/changelog or current time.
#
@ -49,11 +44,14 @@ if (CCACHE_FOUND AND NOT COMPILER_MATCHES_CCACHE)
# - 4.0+ will ignore SOURCE_DATE_EPOCH environment variable.
if (CCACHE_VERSION VERSION_GREATER_EQUAL "4.2")
message(STATUS "ccache is 4.2+ no quirks for SOURCE_DATE_EPOCH required")
set(LAUNCHER ${CCACHE_FOUND})
elseif (CCACHE_VERSION VERSION_GREATER_EQUAL "4.0")
message(STATUS "Ignore SOURCE_DATE_EPOCH for ccache")
set_property (GLOBAL PROPERTY RULE_LAUNCH_COMPILE "env -u SOURCE_DATE_EPOCH")
set_property (GLOBAL PROPERTY RULE_LAUNCH_LINK "env -u SOURCE_DATE_EPOCH")
set(LAUNCHER env -u SOURCE_DATE_EPOCH ${CCACHE_FOUND})
endif()
set (CMAKE_CXX_COMPILER_LAUNCHER ${LAUNCHER} ${CMAKE_CXX_COMPILER_LAUNCHER})
set (CMAKE_C_COMPILER_LAUNCHER ${LAUNCHER} ${CMAKE_C_COMPILER_LAUNCHER})
else ()
message(${RECONFIGURE_MESSAGE_LEVEL} "Not using ${CCACHE_FOUND} ${CCACHE_VERSION} bug: https://bugzilla.samba.org/show_bug.cgi?id=8118")
endif ()

9
cmake/linux/toolchain-x86_64.cmake
View File

@ -14,9 +14,12 @@ set (TOOLCHAIN_PATH "${CMAKE_CURRENT_LIST_DIR}/../../contrib/sysroot/linux-x86_6
set (CMAKE_SYSROOT "${TOOLCHAIN_PATH}/x86_64-linux-gnu/libc")
set (CMAKE_C_FLAGS_INIT "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_CXX_FLAGS_INIT "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_ASM_FLAGS_INIT "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_ASM_FLAGS "${CMAKE_ASM_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (CMAKE_MODULE_LINKER_FLAGS "${CMAKE_MODULE_LINKER_FLAGS} --gcc-toolchain=${TOOLCHAIN_PATH}")
set (HAS_PRE_1970_EXITCODE "0" CACHE STRING "Result from TRY_RUN" FORCE)
set (HAS_PRE_1970_EXITCODE__TRYRUN_OUTPUT "" CACHE STRING "Output from TRY_RUN" FORCE)

8
cmake/target.cmake
View File

@ -42,6 +42,14 @@ if (CMAKE_CROSSCOMPILING)
message (FATAL_ERROR "Trying to cross-compile to unsupported system: ${CMAKE_SYSTEM_NAME}!")
endif ()
if (USE_MUSL)
set (USE_SENTRY OFF CACHE INTERNAL "")
set (ENABLE_ODBC OFF CACHE INTERNAL "")
set (ENABLE_GRPC OFF CACHE INTERNAL "")
set (ENABLE_HDFS OFF CACHE INTERNAL "")
set (ENABLE_EMBEDDED_COMPILER OFF CACHE INTERNAL "")
endif ()
# Don't know why but CXX_STANDARD doesn't work for cross-compilation
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++20")

2
contrib/NuRaft vendored

@ -1 +1 @@
Subproject commit bb69d48e0ee35c87a0f19e509a09a914f71f0cff
Subproject commit ff100a8713146e1ca4b4158dd6cc4eef9af47fc3

2
contrib/abseil-cpp vendored

@ -1 +1 @@
Subproject commit b004a8a02418b83de8b686caa0b0f6e39ac2191f
Subproject commit 215105818dfde3174fe799600bb0f3cae233d0bf

2
contrib/abseil-cpp-cmake/CMakeLists.txt
View File

@ -2,6 +2,8 @@ set(ABSL_ROOT_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
if(NOT EXISTS "${ABSL_ROOT_DIR}/CMakeLists.txt")
message(FATAL_ERROR " submodule third_party/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive")
endif()
set(BUILD_TESTING OFF)
set(ABSL_PROPAGATE_CXX_STD ON)
add_subdirectory("${ABSL_ROOT_DIR}" "${ClickHouse_BINARY_DIR}/contrib/abseil-cpp")
add_library(abseil_swiss_tables INTERFACE)

2
contrib/arrow vendored

@ -1 +1 @@
Subproject commit 078e21bad344747b7656ef2d7a4f7410a0a303eb
Subproject commit aa9a7a698e33e278abe053f4634170b3b026e48e

223
contrib/arrow-cmake/CMakeLists.txt
View File

@ -1,5 +1,21 @@
set (CMAKE_CXX_STANDARD 17)
set(ARROW_VERSION "6.0.1")
string(REGEX MATCH "^[0-9]+\\.[0-9]+\\.[0-9]+" ARROW_BASE_VERSION "${ARROW_VERSION}")
set(ARROW_VERSION_MAJOR "6")
set(ARROW_VERSION_MINOR "0")
set(ARROW_VERSION_PATCH "1")
if(ARROW_VERSION_MAJOR STREQUAL "0")
# Arrow 0.x.y => SO version is "x", full SO version is "x.y.0"
set(ARROW_SO_VERSION "${ARROW_VERSION_MINOR}")
set(ARROW_FULL_SO_VERSION "${ARROW_SO_VERSION}.${ARROW_VERSION_PATCH}.0")
else()
# Arrow 1.x.y => SO version is "10x", full SO version is "10x.y.0"
math(EXPR ARROW_SO_VERSION "${ARROW_VERSION_MAJOR} * 100 + ${ARROW_VERSION_MINOR}")
set(ARROW_FULL_SO_VERSION "${ARROW_SO_VERSION}.${ARROW_VERSION_PATCH}.0")
endif()
# === orc
@ -45,6 +61,9 @@ add_subdirectory(${FLATBUFFERS_SRC_DIR} "${FLATBUFFERS_BINARY_DIR}")
message(STATUS "FLATBUFFERS_LIBRARY: ${FLATBUFFERS_LIBRARY}")
# === hdfs
set(HDFS_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/")
# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features.
# Apple Clang compiler failed to compile this code without specifying c++11 standard.
# As result these compiler features detected as absent. In result it failed to compile orc itself.
@ -66,6 +85,7 @@ configure_file("${ORC_INCLUDE_DIR}/orc/orc-config.hh.in" "${ORC_BUILD_INCLUDE_DI
configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/Adaptor.hh")
# ARROW_ORC + adapters/orc/CMakefiles
set(ORC_SRCS
"${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc"
"${ARROW_SRC_DIR}/arrow/adapters/orc/adapter_util.cc"
@ -102,28 +122,8 @@ set(LIBRARY_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src/arrow")
configure_file("${LIBRARY_DIR}/util/config.h.cmake" "${CMAKE_CURRENT_BINARY_DIR}/cpp/src/arrow/util/config.h")
# arrow/cpp/src/arrow/CMakeLists.txt
# arrow/cpp/src/arrow/CMakeLists.txt (ARROW_SRCS + ARROW_COMPUTE + ARROW_IPC)
set(ARROW_SRCS
"${LIBRARY_DIR}/buffer.cc"
"${LIBRARY_DIR}/builder.cc"
"${LIBRARY_DIR}/chunked_array.cc"
"${LIBRARY_DIR}/compare.cc"
"${LIBRARY_DIR}/datum.cc"
"${LIBRARY_DIR}/device.cc"
"${LIBRARY_DIR}/extension_type.cc"
"${LIBRARY_DIR}/memory_pool.cc"
"${LIBRARY_DIR}/pretty_print.cc"
"${LIBRARY_DIR}/record_batch.cc"
"${LIBRARY_DIR}/result.cc"
"${LIBRARY_DIR}/scalar.cc"
"${LIBRARY_DIR}/sparse_tensor.cc"
"${LIBRARY_DIR}/status.cc"
"${LIBRARY_DIR}/table_builder.cc"
"${LIBRARY_DIR}/table.cc"
"${LIBRARY_DIR}/tensor.cc"
"${LIBRARY_DIR}/type.cc"
"${LIBRARY_DIR}/visitor.cc"
"${LIBRARY_DIR}/array/array_base.cc"
"${LIBRARY_DIR}/array/array_binary.cc"
"${LIBRARY_DIR}/array/array_decimal.cc"
@ -143,25 +143,112 @@ set(ARROW_SRCS
"${LIBRARY_DIR}/array/diff.cc"
"${LIBRARY_DIR}/array/util.cc"
"${LIBRARY_DIR}/array/validate.cc"
"${LIBRARY_DIR}/builder.cc"
"${LIBRARY_DIR}/buffer.cc"
"${LIBRARY_DIR}/chunked_array.cc"
"${LIBRARY_DIR}/compare.cc"
"${LIBRARY_DIR}/config.cc"
"${LIBRARY_DIR}/datum.cc"
"${LIBRARY_DIR}/device.cc"
"${LIBRARY_DIR}/extension_type.cc"
"${LIBRARY_DIR}/memory_pool.cc"
"${LIBRARY_DIR}/pretty_print.cc"
"${LIBRARY_DIR}/record_batch.cc"
"${LIBRARY_DIR}/result.cc"
"${LIBRARY_DIR}/scalar.cc"
"${LIBRARY_DIR}/sparse_tensor.cc"
"${LIBRARY_DIR}/status.cc"
"${LIBRARY_DIR}/table.cc"
"${LIBRARY_DIR}/table_builder.cc"
"${LIBRARY_DIR}/tensor.cc"
"${LIBRARY_DIR}/tensor/coo_converter.cc"
"${LIBRARY_DIR}/tensor/csf_converter.cc"
"${LIBRARY_DIR}/tensor/csx_converter.cc"
"${LIBRARY_DIR}/type.cc"
"${LIBRARY_DIR}/visitor.cc"
"${LIBRARY_DIR}/c/bridge.cc"
"${LIBRARY_DIR}/io/buffered.cc"
"${LIBRARY_DIR}/io/caching.cc"
"${LIBRARY_DIR}/io/compressed.cc"
"${LIBRARY_DIR}/io/file.cc"
"${LIBRARY_DIR}/io/hdfs.cc"
"${LIBRARY_DIR}/io/hdfs_internal.cc"
"${LIBRARY_DIR}/io/interfaces.cc"
"${LIBRARY_DIR}/io/memory.cc"
"${LIBRARY_DIR}/io/slow.cc"
"${LIBRARY_DIR}/io/stdio.cc"
"${LIBRARY_DIR}/io/transform.cc"
"${LIBRARY_DIR}/util/async_util.cc"
"${LIBRARY_DIR}/util/basic_decimal.cc"
"${LIBRARY_DIR}/util/bit_block_counter.cc"
"${LIBRARY_DIR}/util/bit_run_reader.cc"
"${LIBRARY_DIR}/util/bit_util.cc"
"${LIBRARY_DIR}/util/bitmap.cc"
"${LIBRARY_DIR}/util/bitmap_builders.cc"
"${LIBRARY_DIR}/util/bitmap_ops.cc"
"${LIBRARY_DIR}/util/bpacking.cc"
"${LIBRARY_DIR}/util/cancel.cc"
"${LIBRARY_DIR}/util/compression.cc"
"${LIBRARY_DIR}/util/counting_semaphore.cc"
"${LIBRARY_DIR}/util/cpu_info.cc"
"${LIBRARY_DIR}/util/decimal.cc"
"${LIBRARY_DIR}/util/delimiting.cc"
"${LIBRARY_DIR}/util/formatting.cc"
"${LIBRARY_DIR}/util/future.cc"
"${LIBRARY_DIR}/util/int_util.cc"
"${LIBRARY_DIR}/util/io_util.cc"
"${LIBRARY_DIR}/util/logging.cc"
"${LIBRARY_DIR}/util/key_value_metadata.cc"
"${LIBRARY_DIR}/util/memory.cc"
"${LIBRARY_DIR}/util/mutex.cc"
"${LIBRARY_DIR}/util/string.cc"
"${LIBRARY_DIR}/util/string_builder.cc"
"${LIBRARY_DIR}/util/task_group.cc"
"${LIBRARY_DIR}/util/tdigest.cc"
"${LIBRARY_DIR}/util/thread_pool.cc"
"${LIBRARY_DIR}/util/time.cc"
"${LIBRARY_DIR}/util/trie.cc"
"${LIBRARY_DIR}/util/unreachable.cc"
"${LIBRARY_DIR}/util/uri.cc"
"${LIBRARY_DIR}/util/utf8.cc"
"${LIBRARY_DIR}/util/value_parsing.cc"
"${LIBRARY_DIR}/vendored/base64.cpp"
"${LIBRARY_DIR}/vendored/datetime/tz.cpp"
"${LIBRARY_DIR}/vendored/musl/strptime.c"
"${LIBRARY_DIR}/vendored/uriparser/UriCommon.c"
"${LIBRARY_DIR}/vendored/uriparser/UriCompare.c"
"${LIBRARY_DIR}/vendored/uriparser/UriEscape.c"
"${LIBRARY_DIR}/vendored/uriparser/UriFile.c"
"${LIBRARY_DIR}/vendored/uriparser/UriIp4Base.c"
"${LIBRARY_DIR}/vendored/uriparser/UriIp4.c"
"${LIBRARY_DIR}/vendored/uriparser/UriMemory.c"
"${LIBRARY_DIR}/vendored/uriparser/UriNormalizeBase.c"
"${LIBRARY_DIR}/vendored/uriparser/UriNormalize.c"
"${LIBRARY_DIR}/vendored/uriparser/UriParseBase.c"
"${LIBRARY_DIR}/vendored/uriparser/UriParse.c"
"${LIBRARY_DIR}/vendored/uriparser/UriQuery.c"
"${LIBRARY_DIR}/vendored/uriparser/UriRecompose.c"
"${LIBRARY_DIR}/vendored/uriparser/UriResolve.c"
"${LIBRARY_DIR}/vendored/uriparser/UriShorten.c"
"${LIBRARY_DIR}/compute/api_aggregate.cc"
"${LIBRARY_DIR}/compute/api_scalar.cc"
"${LIBRARY_DIR}/compute/api_vector.cc"
"${LIBRARY_DIR}/compute/cast.cc"
"${LIBRARY_DIR}/compute/exec.cc"
"${LIBRARY_DIR}/compute/exec/aggregate_node.cc"
"${LIBRARY_DIR}/compute/exec/exec_plan.cc"
"${LIBRARY_DIR}/compute/exec/expression.cc"
"${LIBRARY_DIR}/compute/exec/filter_node.cc"
"${LIBRARY_DIR}/compute/exec/project_node.cc"
"${LIBRARY_DIR}/compute/exec/source_node.cc"
"${LIBRARY_DIR}/compute/exec/sink_node.cc"
"${LIBRARY_DIR}/compute/exec/order_by_impl.cc"
"${LIBRARY_DIR}/compute/function.cc"
"${LIBRARY_DIR}/compute/function_internal.cc"
"${LIBRARY_DIR}/compute/kernel.cc"
"${LIBRARY_DIR}/compute/registry.cc"
"${LIBRARY_DIR}/compute/exec/exec_plan.cc"
"${LIBRARY_DIR}/compute/exec/expression.cc"
"${LIBRARY_DIR}/compute/exec/key_compare.cc"
"${LIBRARY_DIR}/compute/exec/key_encode.cc"
"${LIBRARY_DIR}/compute/exec/key_hash.cc"
"${LIBRARY_DIR}/compute/exec/key_map.cc"
"${LIBRARY_DIR}/compute/exec/util.cc"
"${LIBRARY_DIR}/compute/kernels/aggregate_basic.cc"
"${LIBRARY_DIR}/compute/kernels/aggregate_mode.cc"
"${LIBRARY_DIR}/compute/kernels/aggregate_quantile.cc"
@ -179,28 +266,31 @@ set(ARROW_SRCS
"${LIBRARY_DIR}/compute/kernels/scalar_cast_string.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_cast_temporal.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_compare.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_fill_null.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_if_else.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_nested.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_set_lookup.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_string.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_temporal.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_temporal_binary.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_temporal_unary.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_validity.cc"
"${LIBRARY_DIR}/compute/kernels/scalar_if_else.cc"
"${LIBRARY_DIR}/compute/kernels/util_internal.cc"
"${LIBRARY_DIR}/compute/kernels/vector_array_sort.cc"
"${LIBRARY_DIR}/compute/kernels/vector_hash.cc"
"${LIBRARY_DIR}/compute/kernels/vector_nested.cc"
"${LIBRARY_DIR}/compute/kernels/vector_replace.cc"
"${LIBRARY_DIR}/compute/kernels/vector_selection.cc"
"${LIBRARY_DIR}/compute/kernels/vector_sort.cc"
"${LIBRARY_DIR}/csv/chunker.cc"
"${LIBRARY_DIR}/csv/column_builder.cc"
"${LIBRARY_DIR}/csv/column_decoder.cc"
"${LIBRARY_DIR}/csv/converter.cc"
"${LIBRARY_DIR}/csv/options.cc"
"${LIBRARY_DIR}/csv/parser.cc"
"${LIBRARY_DIR}/csv/reader.cc"
"${LIBRARY_DIR}/csv/writer.cc"
"${LIBRARY_DIR}/compute/kernels/row_encoder.cc"
"${LIBRARY_DIR}/compute/exec/union_node.cc"
"${LIBRARY_DIR}/compute/exec/key_hash.cc"
"${LIBRARY_DIR}/compute/exec/key_map.cc"
"${LIBRARY_DIR}/compute/exec/key_compare.cc"
"${LIBRARY_DIR}/compute/exec/key_encode.cc"
"${LIBRARY_DIR}/compute/exec/util.cc"
"${LIBRARY_DIR}/compute/exec/hash_join_dict.cc"
"${LIBRARY_DIR}/compute/exec/hash_join.cc"
"${LIBRARY_DIR}/compute/exec/hash_join_node.cc"
"${LIBRARY_DIR}/compute/exec/task_util.cc"
"${LIBRARY_DIR}/ipc/dictionary.cc"
"${LIBRARY_DIR}/ipc/feather.cc"
@ -210,52 +300,6 @@ set(ARROW_SRCS
"${LIBRARY_DIR}/ipc/reader.cc"
"${LIBRARY_DIR}/ipc/writer.cc"
"${LIBRARY_DIR}/io/buffered.cc"
"${LIBRARY_DIR}/io/caching.cc"
"${LIBRARY_DIR}/io/compressed.cc"
"${LIBRARY_DIR}/io/file.cc"
"${LIBRARY_DIR}/io/interfaces.cc"
"${LIBRARY_DIR}/io/memory.cc"
"${LIBRARY_DIR}/io/slow.cc"
"${LIBRARY_DIR}/io/stdio.cc"
"${LIBRARY_DIR}/io/transform.cc"
"${LIBRARY_DIR}/tensor/coo_converter.cc"
"${LIBRARY_DIR}/tensor/csf_converter.cc"
"${LIBRARY_DIR}/tensor/csx_converter.cc"
"${LIBRARY_DIR}/util/basic_decimal.cc"
"${LIBRARY_DIR}/util/bit_block_counter.cc"
"${LIBRARY_DIR}/util/bit_run_reader.cc"
"${LIBRARY_DIR}/util/bit_util.cc"
"${LIBRARY_DIR}/util/bitmap_builders.cc"
"${LIBRARY_DIR}/util/bitmap_ops.cc"
"${LIBRARY_DIR}/util/bitmap.cc"
"${LIBRARY_DIR}/util/bpacking.cc"
"${LIBRARY_DIR}/util/cancel.cc"
"${LIBRARY_DIR}/util/compression.cc"
"${LIBRARY_DIR}/util/cpu_info.cc"
"${LIBRARY_DIR}/util/decimal.cc"
"${LIBRARY_DIR}/util/delimiting.cc"
"${LIBRARY_DIR}/util/formatting.cc"
"${LIBRARY_DIR}/util/future.cc"
"${LIBRARY_DIR}/util/int_util.cc"
"${LIBRARY_DIR}/util/io_util.cc"
"${LIBRARY_DIR}/util/key_value_metadata.cc"
"${LIBRARY_DIR}/util/logging.cc"
"${LIBRARY_DIR}/util/memory.cc"
"${LIBRARY_DIR}/util/mutex.cc"
"${LIBRARY_DIR}/util/string_builder.cc"
"${LIBRARY_DIR}/util/string.cc"
"${LIBRARY_DIR}/util/task_group.cc"
"${LIBRARY_DIR}/util/tdigest.cc"
"${LIBRARY_DIR}/util/thread_pool.cc"
"${LIBRARY_DIR}/util/time.cc"
"${LIBRARY_DIR}/util/trie.cc"
"${LIBRARY_DIR}/util/utf8.cc"
"${LIBRARY_DIR}/util/value_parsing.cc"
"${LIBRARY_DIR}/vendored/base64.cpp"
${ORC_SRCS}
)
@ -325,6 +369,7 @@ target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_BUILD_INCLUDE_D
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ORC_ADDITION_SOURCE_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${ARROW_SRC_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${FLATBUFFERS_INCLUDE_DIR})
target_include_directories(${ARROW_LIBRARY} PRIVATE SYSTEM ${HDFS_INCLUDE_DIR})
# === parquet
@ -399,7 +444,7 @@ set (HAVE_STRERROR_R 1)
set (HAVE_SCHED_GET_PRIORITY_MAX 1)
set (HAVE_SCHED_GET_PRIORITY_MIN 1)
if (OS_LINUX)
if (OS_LINUX AND NOT USE_MUSL)
set (STRERROR_R_CHAR_P 1)
endif ()

2
contrib/boost vendored

@ -1 +1 @@
Subproject commit fcb058e1459ac273ecfe7cdf72791cb1479115af
Subproject commit c0807e83f2824e8dd67a15b355496a9b784cdcd5

22
contrib/boost-cmake/CMakeLists.txt
View File

@ -1,9 +1,7 @@
option (USE_INTERNAL_BOOST_LIBRARY "Use internal Boost library" ON)
if (NOT USE_INTERNAL_BOOST_LIBRARY)
# 1.70 like in contrib/boost
# 1.71 on CI
set(BOOST_VERSION 1.71)
set(BOOST_VERSION 1.78)
find_package(Boost ${BOOST_VERSION} COMPONENTS
system
@ -66,9 +64,11 @@ if (NOT EXTERNAL_BOOST_FOUND)
set (SRCS_FILESYSTEM
"${LIBRARY_DIR}/libs/filesystem/src/codecvt_error_category.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/directory.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/exception.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/operations.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/path_traits.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/path.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/path_traits.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/portability.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/unique_path.cpp"
"${LIBRARY_DIR}/libs/filesystem/src/utf8_codecvt_facet.cpp"
@ -126,24 +126,11 @@ if (NOT EXTERNAL_BOOST_FOUND)
# regex
set (SRCS_REGEX
"${LIBRARY_DIR}/libs/regex/src/c_regex_traits.cpp"
"${LIBRARY_DIR}/libs/regex/src/cpp_regex_traits.cpp"
"${LIBRARY_DIR}/libs/regex/src/cregex.cpp"
"${LIBRARY_DIR}/libs/regex/src/fileiter.cpp"
"${LIBRARY_DIR}/libs/regex/src/icu.cpp"
"${LIBRARY_DIR}/libs/regex/src/instances.cpp"
"${LIBRARY_DIR}/libs/regex/src/internals.hpp"
"${LIBRARY_DIR}/libs/regex/src/posix_api.cpp"
"${LIBRARY_DIR}/libs/regex/src/regex_debug.cpp"
"${LIBRARY_DIR}/libs/regex/src/regex_raw_buffer.cpp"
"${LIBRARY_DIR}/libs/regex/src/regex_traits_defaults.cpp"
"${LIBRARY_DIR}/libs/regex/src/regex.cpp"
"${LIBRARY_DIR}/libs/regex/src/static_mutex.cpp"
"${LIBRARY_DIR}/libs/regex/src/usinstances.cpp"
"${LIBRARY_DIR}/libs/regex/src/w32_regex_traits.cpp"
"${LIBRARY_DIR}/libs/regex/src/wc_regex_traits.cpp"
"${LIBRARY_DIR}/libs/regex/src/wide_posix_api.cpp"
"${LIBRARY_DIR}/libs/regex/src/winstances.cpp"
)
add_library (_boost_regex ${SRCS_REGEX})
@ -166,7 +153,6 @@ if (NOT EXTERNAL_BOOST_FOUND)
set (SRCS_CONTEXT
"${LIBRARY_DIR}/libs/context/src/dummy.cpp"
"${LIBRARY_DIR}/libs/context/src/execution_context.cpp"
"${LIBRARY_DIR}/libs/context/src/posix/stack_traits.cpp"
)

2
contrib/cassandra vendored

@ -1 +1 @@
Subproject commit eb9b68dadbb4417a2c132ad4a1c2fa76e65e6fc1
Subproject commit f4a31e92a25c34c02c7291ff97c7813bc83b0e09

2
contrib/jemalloc vendored

@ -1 +1 @@
Subproject commit e6891d9746143bf2cf617493d880ba5a0b9a3efd
Subproject commit a1404807211b1612539f840b3dcb1bf38d1a269e

11
contrib/libuv-cmake/CMakeLists.txt
View File

@ -1,17 +1,8 @@
# This file is a modified version of contrib/libuv/CMakeLists.txt
include(CMakeDependentOption)
set (SOURCE_DIR "${CMAKE_SOURCE_DIR}/contrib/libuv")
set (BINARY_DIR "${CMAKE_BINARY_DIR}/contrib/libuv")
if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU")
list(APPEND uv_cflags -fvisibility=hidden --std=gnu89)
list(APPEND uv_cflags -Wall -Wextra -Wstrict-prototypes)
list(APPEND uv_cflags -Wno-unused-parameter)
endif()
set(uv_sources
src/fs-poll.c
src/idna.c
@ -76,7 +67,7 @@ endif()
if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
list(APPEND uv_defines _GNU_SOURCE _POSIX_C_SOURCE=200112)
list(APPEND uv_libraries dl rt)
list(APPEND uv_libraries rt)
list(APPEND uv_sources
src/unix/linux-core.c
src/unix/linux-inotify.c

5
contrib/mariadb-connector-c-cmake/CMakeLists.txt
View File

@ -236,8 +236,7 @@ set(LIBMARIADB_SOURCES ${LIBMARIADB_SOURCES} ${CC_SOURCE_DIR}/libmariadb/mariadb
add_library(mariadbclient STATIC ${LIBMARIADB_SOURCES})
target_link_libraries(mariadbclient ${SYSTEM_LIBS})
target_include_directories(mariadbclient
PRIVATE ${CC_BINARY_DIR}/include-private
PUBLIC ${CC_BINARY_DIR}/include-public ${CC_SOURCE_DIR}/include ${CC_SOURCE_DIR}/libmariadb)
target_include_directories(mariadbclient PRIVATE ${CC_BINARY_DIR}/include-private)
target_include_directories(mariadbclient SYSTEM PUBLIC ${CC_BINARY_DIR}/include-public ${CC_SOURCE_DIR}/include ${CC_SOURCE_DIR}/libmariadb)
set_target_properties(mariadbclient PROPERTIES IMPORTED_INTERFACE_LINK_LIBRARIES "${SYSTEM_LIBS}")

2
contrib/protobuf vendored

@ -1 +1 @@
Subproject commit c1c5d02026059f4c3cb51aaa08e82288d3e08b89
Subproject commit 6bb70196c5360268d9f021bb7936fb0b551724c2

2
contrib/s2geometry vendored

@ -1 +1 @@
Subproject commit 38b7a290f927cc372218c2094602b83e35b18c05
Subproject commit 471fe9dc931a4bb560333545186e9b5da168ac83

58
contrib/s2geometry-cmake/CMakeLists.txt
View File

@ -1,8 +1,12 @@
set(S2_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/s2geometry/src")
set(ABSL_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/abseil-cpp")
if(NOT EXISTS "${ABSL_SOURCE_DIR}/CMakeLists.txt")
message(FATAL_ERROR " submodule contrib/abseil-cpp is missing. To fix try run: \n git submodule update --init --recursive")
endif()
set(S2_SRCS
"${S2_SOURCE_DIR}/s2/base/stringprintf.cc"
"${S2_SOURCE_DIR}/s2/base/strtoint.cc"
"${S2_SOURCE_DIR}/s2/encoded_s2cell_id_vector.cc"
"${S2_SOURCE_DIR}/s2/encoded_s2point_vector.cc"
"${S2_SOURCE_DIR}/s2/encoded_s2shape_index.cc"
@ -14,11 +18,14 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s1chord_angle.cc"
"${S2_SOURCE_DIR}/s2/s1interval.cc"
"${S2_SOURCE_DIR}/s2/s2boolean_operation.cc"
"${S2_SOURCE_DIR}/s2/s2buffer_operation.cc"
"${S2_SOURCE_DIR}/s2/s2builder.cc"
"${S2_SOURCE_DIR}/s2/s2builder_graph.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_closed_set_normalizer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_find_polygon_degeneracies.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_get_snapped_winding_delta.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_lax_polygon_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_lax_polyline_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_s2point_vector_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_s2polygon_layer.cc"
"${S2_SOURCE_DIR}/s2/s2builderutil_s2polyline_layer.cc"
@ -44,7 +51,6 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2edge_crossings.cc"
"${S2_SOURCE_DIR}/s2/s2edge_distances.cc"
"${S2_SOURCE_DIR}/s2/s2edge_tessellator.cc"
"${S2_SOURCE_DIR}/s2/s2error.cc"
"${S2_SOURCE_DIR}/s2/s2furthest_edge_query.cc"
"${S2_SOURCE_DIR}/s2/s2latlng.cc"
"${S2_SOURCE_DIR}/s2/s2latlng_rect.cc"
@ -55,6 +61,7 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2loop.cc"
"${S2_SOURCE_DIR}/s2/s2loop_measures.cc"
"${S2_SOURCE_DIR}/s2/s2measures.cc"
"${S2_SOURCE_DIR}/s2/s2memory_tracker.cc"
"${S2_SOURCE_DIR}/s2/s2metrics.cc"
"${S2_SOURCE_DIR}/s2/s2max_distance_targets.cc"
"${S2_SOURCE_DIR}/s2/s2min_distance_targets.cc"
@ -82,28 +89,15 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/s2shapeutil_build_polygon_boundaries.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_coding.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_contains_brute_force.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_conversion.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_edge_iterator.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_get_reference_point.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_range_iterator.cc"
"${S2_SOURCE_DIR}/s2/s2shapeutil_visit_crossing_edge_pairs.cc"
"${S2_SOURCE_DIR}/s2/s2text_format.cc"
"${S2_SOURCE_DIR}/s2/s2wedge_relations.cc"
"${S2_SOURCE_DIR}/s2/strings/ostringstream.cc"
"${S2_SOURCE_DIR}/s2/s2winding_operation.cc"
"${S2_SOURCE_DIR}/s2/strings/serialize.cc"
# ClickHouse doesn't use strings from abseil.
# So, there is no duplicate symbols.
"${S2_SOURCE_DIR}/s2/third_party/absl/base/dynamic_annotations.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/base/internal/raw_logging.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/base/internal/throw_delegate.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/numeric/int128.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/ascii.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/match.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/numbers.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/str_cat.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/str_split.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/string_view.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/strip.cc"
"${S2_SOURCE_DIR}/s2/third_party/absl/strings/internal/memutil.cc"
"${S2_SOURCE_DIR}/s2/util/bits/bit-interleave.cc"
"${S2_SOURCE_DIR}/s2/util/bits/bits.cc"
"${S2_SOURCE_DIR}/s2/util/coding/coder.cc"
@ -111,17 +105,41 @@ set(S2_SRCS
"${S2_SOURCE_DIR}/s2/util/math/exactfloat/exactfloat.cc"
"${S2_SOURCE_DIR}/s2/util/math/mathutil.cc"
"${S2_SOURCE_DIR}/s2/util/units/length-units.cc"
)
add_library(s2 ${S2_SRCS})
set_property(TARGET s2 PROPERTY CXX_STANDARD 11)
set_property(TARGET s2 PROPERTY CXX_STANDARD 17)
if (OPENSSL_FOUND)
target_link_libraries(s2 PRIVATE ${OPENSSL_LIBRARIES})
endif()
# Copied from contrib/s2geometry/CMakeLists
target_link_libraries(s2 PRIVATE
absl::base
absl::btree
absl::config
absl::core_headers
absl::dynamic_annotations
absl::endian
absl::fixed_array
absl::flat_hash_map
absl::flat_hash_set
absl::hash
absl::inlined_vector
absl::int128
absl::log_severity
absl::memory
absl::span
absl::str_format
absl::strings
absl::type_traits
absl::utility
)
target_include_directories(s2 SYSTEM BEFORE PUBLIC "${S2_SOURCE_DIR}/")
target_include_directories(s2 SYSTEM PUBLIC "${ABSL_SOURCE_DIR}")
if(M_LIBRARY)
target_link_libraries(s2 PRIVATE ${M_LIBRARY})

2
contrib/sysroot vendored

@ -1 +1 @@
Subproject commit 410845187f582c5e6692b53dddbe43efbb728734
Subproject commit bbcac834526d90d1e764164b861be426891d1743

5
docker/images.json
View File

@ -46,7 +46,6 @@
"name": "clickhouse/stateless-test",
"dependent": [
"docker/test/stateful",
"docker/test/coverage",
"docker/test/unit"
]
},
@ -56,10 +55,6 @@
"docker/test/stress"
]
},
"docker/test/coverage": {
"name": "clickhouse/test-coverage",
"dependent": []
},
"docker/test/unit": {
"name": "clickhouse/unit-test",
"dependent": []

6
docker/test/codebrowser/Dockerfile
View File

@ -6,7 +6,7 @@ FROM clickhouse/binary-builder
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-9 libllvm9 libclang-9-dev
RUN apt-get update && apt-get --yes --allow-unauthenticated install clang-13 libllvm13 libclang-13-dev
# repo versions doesn't work correctly with C++17
# also we push reports to s3, so we add index.html to subfolder urls
@ -23,12 +23,12 @@ ENV SOURCE_DIRECTORY=/repo_folder
ENV BUILD_DIRECTORY=/build
ENV HTML_RESULT_DIRECTORY=$BUILD_DIRECTORY/html_report
ENV SHA=nosha
ENV DATA="data"
ENV DATA="https://s3.amazonaws.com/clickhouse-test-reports/codebrowser/data"
CMD mkdir -p $BUILD_DIRECTORY && cd $BUILD_DIRECTORY && \
cmake $SOURCE_DIRECTORY -DCMAKE_CXX_COMPILER=/usr/bin/clang\+\+-13 -DCMAKE_C_COMPILER=/usr/bin/clang-13 -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DENABLE_EMBEDDED_COMPILER=0 -DENABLE_S3=0 && \
mkdir -p $HTML_RESULT_DIRECTORY && \
$CODEGEN -b $BUILD_DIRECTORY -a -o $HTML_RESULT_DIRECTORY -p ClickHouse:$SOURCE_DIRECTORY:$SHA -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \
cp -r $STATIC_DATA $HTML_RESULT_DIRECTORY/ &&\
$CODEINDEX $HTML_RESULT_DIRECTORY -d $DATA | ts '%Y-%m-%d %H:%M:%S' && \
$CODEINDEX $HTML_RESULT_DIRECTORY -d "$DATA" | ts '%Y-%m-%d %H:%M:%S' && \
mv $HTML_RESULT_DIRECTORY /test_output

18
docker/test/coverage/Dockerfile
View File

@ -1,18 +0,0 @@
# docker build -t clickhouse/test-coverage .
FROM clickhouse/stateless-test
RUN apt-get update -y \
&& env DEBIAN_FRONTEND=noninteractive \
apt-get install --yes --no-install-recommends \
cmake
COPY s3downloader /s3downloader
COPY run.sh /run.sh
ENV DATASETS="hits visits"
ENV COVERAGE_DIR=/coverage_reports
ENV SOURCE_DIR=/build
ENV OUTPUT_DIR=/output
ENV IGNORE='.*contrib.*'
CMD ["/bin/bash", "/run.sh"]

112
docker/test/coverage/run.sh
View File

@ -1,112 +0,0 @@
#!/bin/bash
kill_clickhouse () {
echo "clickhouse pids $(pgrep -u clickhouse)" | ts '%Y-%m-%d %H:%M:%S'
pkill -f "clickhouse-server" 2>/dev/null
for _ in {1..120}
do
if ! pkill -0 -f "clickhouse-server" ; then break ; fi
echo "ClickHouse still alive" | ts '%Y-%m-%d %H:%M:%S'
sleep 1
done
if pkill -0 -f "clickhouse-server"
then
pstree -apgT
jobs
echo "Failed to kill the ClickHouse server" | ts '%Y-%m-%d %H:%M:%S'
return 1
fi
}
start_clickhouse () {
LLVM_PROFILE_FILE='server_%h_%p_%m.profraw' sudo -Eu clickhouse /usr/bin/clickhouse-server --config /etc/clickhouse-server/config.xml &
counter=0
until clickhouse-client --query "SELECT 1"
do
if [ "$counter" -gt 120 ]
then
echo "Cannot start clickhouse-server"
cat /var/log/clickhouse-server/stdout.log
tail -n1000 /var/log/clickhouse-server/stderr.log
tail -n1000 /var/log/clickhouse-server/clickhouse-server.log
break
fi
sleep 0.5
counter=$((counter + 1))
done
}
chmod 777 /
dpkg -i package_folder/clickhouse-common-static_*.deb; \
dpkg -i package_folder/clickhouse-common-static-dbg_*.deb; \
dpkg -i package_folder/clickhouse-server_*.deb; \
dpkg -i package_folder/clickhouse-client_*.deb; \
dpkg -i package_folder/clickhouse-test_*.deb
mkdir -p /var/lib/clickhouse
mkdir -p /var/log/clickhouse-server
chmod 777 -R /var/log/clickhouse-server/
# install test configs
/usr/share/clickhouse-test/config/install.sh
start_clickhouse
# shellcheck disable=SC2086 # No quotes because I want to split it into words.
if ! /s3downloader --dataset-names $DATASETS; then
echo "Cannot download datatsets"
exit 1
fi
chmod 777 -R /var/lib/clickhouse
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW DATABASES"
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "ATTACH DATABASE datasets ENGINE = Ordinary"
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "CREATE DATABASE test"
kill_clickhouse
start_clickhouse
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM datasets"
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-client --query "SHOW TABLES FROM test"
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test -j 8 --testname --shard --zookeeper --print-time 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee /test_result.txt
readarray -t FAILED_TESTS < <(awk '/FAIL|TIMEOUT|ERROR/ { print substr($3, 1, length($3)-1) }' "/test_result.txt")
kill_clickhouse
sleep 3
if [[ -n "${FAILED_TESTS[*]}" ]]
then
# Clean the data so that there is no interference from the previous test run.
rm -rf /var/lib/clickhouse/{{meta,}data,user_files} ||:
start_clickhouse
echo "Going to run again: ${FAILED_TESTS[*]}"
LLVM_PROFILE_FILE='client_coverage_%5m.profraw' clickhouse-test --order=random --testname --shard --zookeeper "${FAILED_TESTS[@]}" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee -a /test_result.txt
else
echo "No failed tests"
fi
mkdir -p "$COVERAGE_DIR"
mv /*.profraw "$COVERAGE_DIR"
mkdir -p "$SOURCE_DIR"/obj-x86_64-linux-gnu
cd "$SOURCE_DIR"/obj-x86_64-linux-gnu && CC=clang-11 CXX=clang++-11 cmake .. && cd /
llvm-profdata-11 merge -sparse "${COVERAGE_DIR}"/* -o clickhouse.profdata
llvm-cov-11 export /usr/bin/clickhouse -instr-profile=clickhouse.profdata -j=16 -format=lcov -skip-functions -ignore-filename-regex "$IGNORE" > output.lcov
genhtml output.lcov --ignore-errors source --output-directory "${OUTPUT_DIR}"

101
docker/test/coverage/s3downloader
View File

@ -1,101 +0,0 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import sys
import time
import tarfile
import logging
import argparse
import requests
import tempfile
DEFAULT_URL = 'https://clickhouse-datasets.s3.yandex.net'
AVAILABLE_DATASETS = {
'hits': 'hits_v1.tar',
'visits': 'visits_v1.tar',
}
RETRIES_COUNT = 5
def _get_temp_file_name():
return os.path.join(tempfile._get_default_tempdir(), next(tempfile._get_candidate_names()))
def build_url(base_url, dataset):
return os.path.join(base_url, dataset, 'partitions', AVAILABLE_DATASETS[dataset])
def dowload_with_progress(url, path):
logging.info("Downloading from %s to temp path %s", url, path)
for i in range(RETRIES_COUNT):
try:
with open(path, 'wb') as f:
response = requests.get(url, stream=True)
response.raise_for_status()
total_length = response.headers.get('content-length')
if total_length is None or int(total_length) == 0:
logging.info("No content-length, will download file without progress")
f.write(response.content)
else:
dl = 0
total_length = int(total_length)
logging.info("Content length is %ld bytes", total_length)
for data in response.iter_content(chunk_size=4096):
dl += len(data)
f.write(data)
if sys.stdout.isatty():
done = int(50 * dl / total_length)
percent = int(100 * float(dl) / total_length)
sys.stdout.write("\r[{}{}] {}%".format('=' * done, ' ' * (50-done), percent))
sys.stdout.flush()
break
except Exception as ex:
sys.stdout.write("\n")
time.sleep(3)
logging.info("Exception while downloading %s, retry %s", ex, i + 1)
if os.path.exists(path):
os.remove(path)
else:
raise Exception("Cannot download dataset from {}, all retries exceeded".format(url))
sys.stdout.write("\n")
logging.info("Downloading finished")
def unpack_to_clickhouse_directory(tar_path, clickhouse_path):
logging.info("Will unpack data from temp path %s to clickhouse db %s", tar_path, clickhouse_path)
with tarfile.open(tar_path, 'r') as comp_file:
comp_file.extractall(path=clickhouse_path)
logging.info("Unpack finished")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
parser = argparse.ArgumentParser(
description="Simple tool for dowloading datasets for clickhouse from S3")
parser.add_argument('--dataset-names', required=True, nargs='+', choices=list(AVAILABLE_DATASETS.keys()))
parser.add_argument('--url-prefix', default=DEFAULT_URL)
parser.add_argument('--clickhouse-data-path', default='/var/lib/clickhouse/')
args = parser.parse_args()
datasets = args.dataset_names
logging.info("Will fetch following datasets: %s", ', '.join(datasets))
for dataset in datasets:
logging.info("Processing %s", dataset)
temp_archive_path = _get_temp_file_name()
try:
download_url_for_dataset = build_url(args.url_prefix, dataset)
dowload_with_progress(download_url_for_dataset, temp_archive_path)
unpack_to_clickhouse_directory(temp_archive_path, args.clickhouse_data_path)
except Exception as ex:
logging.info("Some exception occured %s", str(ex))
raise
finally:
logging.info("Will remove downloaded file %s from filesystem if it exists", temp_archive_path)
if os.path.exists(temp_archive_path):
os.remove(temp_archive_path)
logging.info("Processing of %s finished", dataset)
logging.info("Fetch finished, enjoy your tables!")

8
docker/test/fasttest/run.sh
View File

@ -257,7 +257,13 @@ function run_tests
start_server
set +e
time clickhouse-test --hung-check -j 8 --order=random \
local NPROC
NPROC=$(nproc)
NPROC=$((NPROC / 2))
if [[ $NPROC == 0 ]]; then
NPROC=1
fi
time clickhouse-test --hung-check -j "${NPROC}" --order=random \
--fast-tests-only --no-long --testname --shard --zookeeper --check-zookeeper-session \
-- "$FASTTEST_FOCUS" 2>&1 \
| ts '%Y-%m-%d %H:%M:%S' \

4
docker/test/fuzzer/run-fuzzer.sh
View File

@ -194,8 +194,8 @@ quit
time clickhouse-client --query "SELECT 'Connected to clickhouse-server after attaching gdb'" ||:
# Check connectivity after we attach gdb, because it might cause the server
# to freeze and the fuzzer will fail.
for _ in {1..60}
# to freeze and the fuzzer will fail. In debug build it can take a lot of time.
for _ in {1..180}
do
sleep 1
if clickhouse-client --query "select 1"

2
docker/test/pvs/Dockerfile
View File

@ -42,7 +42,7 @@ ENV CCACHE_DIR=/test_output/ccache
CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"DISABLE_HERMETIC_BUILD"=ON -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
&& ninja re2_st clickhouse_grpc_protos \
&& pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
&& pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j "$(nproc)" -l ./licence.lic; \
cp /repo_folder/pvs-studio.log /test_output; \
plog-converter -a GA:1,2 -t fullhtml -o /test_output/pvs-studio-html-report pvs-studio.log; \
plog-converter -a GA:1,2 -t tasklist -o /test_output/pvs-studio-task-report.txt pvs-studio.log

75
docs/en/engines/database-engines/materialized-mysql.md
View File

@ -83,6 +83,7 @@ When working with the `MaterializedMySQL` database engine, [ReplacingMergeTree](
| VARCHAR, VAR_STRING | [String](../../sql-reference/data-types/string.md) |
| BLOB | [String](../../sql-reference/data-types/string.md) |
| BINARY | [FixedString](../../sql-reference/data-types/fixedstring.md) |
| BIT | [UInt64](../../sql-reference/data-types/int-uint.md) |
[Nullable](../../sql-reference/data-types/nullable.md) is supported.
@ -150,20 +151,38 @@ Table overrides can be used to customize the ClickHouse DDL queries, allowing yo
application. This is especially useful for controlling partitioning, which is important for the overall performance of
MaterializedMySQL.
These are the schema conversion manipulations you can do with table overrides for MaterializedMySQL:
* Modify column type. Must be compatible with the original type, or replication will fail. For example,
you can modify a UInt32 column to UInt64, but you can not modify a String column to Array(String).
* Modify [column TTL](../table-engines/mergetree-family/mergetree/#mergetree-column-ttl).
* Modify [column compression codec](../../sql-reference/statements/create/table/#codecs).
* Add [ALIAS columns](../../sql-reference/statements/create/table/#alias).
* Add [skipping indexes](../table-engines/mergetree-family/mergetree/#table_engine-mergetree-data_skipping-indexes)
* Add [projections](../table-engines/mergetree-family/mergetree/#projections). Note that projection optimizations are
disabled when using `SELECT ... FINAL` (which MaterializedMySQL does by default), so their utility is limited here.
`INDEX ... TYPE hypothesis` as [described in the v21.12 blog post]](https://clickhouse.com/blog/en/2021/clickhouse-v21.12-released/)
may be more useful in this case.
* Modify [PARTITION BY](../table-engines/mergetree-family/custom-partitioning-key/)
* Modify [ORDER BY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
* Modify [PRIMARY KEY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
* Add [SAMPLE BY](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
* Add [table TTL](../table-engines/mergetree-family/mergetree/#mergetree-query-clauses)
```sql
CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
[SETTINGS ...]
[TABLE OVERRIDE table_name (
[COLUMNS (
[name1 [type1] [DEFAULT|MATERIALIZED|ALIAS expr1] [TTL expr1], ...]
[INDEX index_name1 expr1 TYPE type1(...) GRANULARITY value1, ...]
[PROJECTION projection_name_1 (SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]), ...]
)]
[ORDER BY expr]
[PRIMARY KEY expr]
[PARTITION BY expr]
[SAMPLE BY expr]
[TTL expr]
[col_name [datatype] [ALIAS expr] [CODEC(...)] [TTL expr], ...]
[INDEX index_name expr TYPE indextype[(...)] GRANULARITY val, ...]
[PROJECTION projection_name (SELECT <COLUMN LIST EXPR> [GROUP BY] [ORDER BY]), ...]
)]
[ORDER BY expr]
[PRIMARY KEY expr]
[PARTITION BY expr]
[SAMPLE BY expr]
[TTL expr]
), ...]
```
@ -173,34 +192,34 @@ Example:
CREATE DATABASE db_name ENGINE = MaterializedMySQL(...)
TABLE OVERRIDE table1 (
COLUMNS (
userid UUID,
category LowCardinality(String),
timestamp DateTime CODEC(Delta, Default)
userid UUID,
category LowCardinality(String),
timestamp DateTime CODEC(Delta, Default)
)
PARTITION BY toYear(timestamp)
),
TABLE OVERRIDE table2 (
COLUMNS (
ip_hash UInt32 MATERIALIZED xxHash32(client_ip),
client_ip String TTL created + INTERVAL 72 HOUR
)
SAMPLE BY ip_hash
client_ip String TTL created + INTERVAL 72 HOUR
)
SAMPLE BY ip_hash
)
```
The `COLUMNS` list is sparse; it contains only modified or extra (MATERIALIZED or ALIAS) columns. Modified columns with
a different type must be assignable from the original type. There is currently no validation of this or similar issues
when the `CREATE DATABASE` query executes, so extra care needs to be taken.
The `COLUMNS` list is sparse; existing columns are modified as specified, extra ALIAS columns are added. It is not
possible to add ordinary or MATERIALIZED columns. Modified columns with a different type must be assignable from the
original type. There is currently no validation of this or similar issues when the `CREATE DATABASE` query executes, so
extra care needs to be taken.
You may specify overrides for tables that do not exist yet.
!!! note "Warning"
It is easy to break replication with TABLE OVERRIDEs if not used with care. For example:
!!! warning "Warning"
It is easy to break replication with table overrides if not used with care. For example:
* If a column is added with a table override, but then later added to the source MySQL table, the converted ALTER TABLE
query in ClickHouse will fail because the column already exists.
* If an ALIAS column is added with a table override, and a column with the same name is later added to the source
MySQL table, the converted ALTER TABLE query in ClickHouse will fail and replication stops.
* It is currently possible to add overrides that reference nullable columns where not-nullable are required, such as in
`ORDER BY` or `PARTITION BY`.
`ORDER BY` or `PARTITION BY`. This will cause CREATE TABLE queries that will fail, also causing replication to stop.
## Examples of Use {#examples-of-use}
@ -217,11 +236,9 @@ mysql> SELECT * FROM test;
```
```text
+---+------+------+
| a | b | c |
+---+------+------+
| 2 | 222 | Wow! |
+---+------+------+
┌─a─┬───b─┬─c────┐
│ 2 │ 222 │ Wow! │
└───┴─────┴──────┘
```
Database in ClickHouse, exchanging data with the MySQL server:

140
docs/en/engines/database-engines/materialized-postgresql.md
View File

@ -5,15 +5,15 @@ toc_title: MaterializedPostgreSQL
# [experimental] MaterializedPostgreSQL {#materialize-postgresql}
Creates ClickHouse database with an initial data dump of PostgreSQL database tables and starts replication process, i.e. executes background job to apply new changes as they happen on PostgreSQL database tables in the remote PostgreSQL database.
Creates a ClickHouse database with tables from PostgreSQL database. Firstly, database with engine `MaterializedPostgreSQL` creates a snapshot of PostgreSQL database and loads required tables. Required tables can include any subset of tables from any subset of schemas from specified database. Along with the snapshot database engine acquires LSN and once initial dump of tables is performed - it starts pulling updates from WAL. After database is created, newly added tables to PostgreSQL database are not automatically added to replication. They have to be added manually with `ATTACH TABLE db.table` query.
ClickHouse server works as PostgreSQL replica. It reads WAL and performs DML queries. DDL is not replicated, but can be handled (described below).
Replication is implemented with PostgreSQL Logical Replication Protocol, which does not allow to replicate DDL, but allows to know whether replication breaking changes happened (column type changes, adding/removing columns). Such changes are detected and according tables stop receiving updates. Such tables can be automatically reloaded in the background in case required setting is turned on. Safest way for now is to use `ATTACH`/ `DETACH` queries to reload table completely. If DDL does not break replication (for example, renaming a column) table will still receive updates (insertion is done by position).
## Creating a Database {#creating-a-database}
``` sql
CREATE DATABASE [IF NOT EXISTS] db_name [ON CLUSTER cluster]
ENGINE = MaterializedPostgreSQL('host:port', ['database' | database], 'user', 'password') [SETTINGS ...]
ENGINE = MaterializedPostgreSQL('host:port', 'database', 'user', 'password') [SETTINGS ...]
```
**Engine Parameters**
@ -23,51 +23,39 @@ ENGINE = MaterializedPostgreSQL('host:port', ['database' | database], 'user', 'p
- `user` — PostgreSQL user.
- `password` — User password.
## Example of Use {#example-of-use}
``` sql
CREATE DATABASE postgresql;
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password');
SHOW TABLES FROM postgres_db;
┌─name───┐
│ table1 │
└────────┘
SELECT * FROM postgresql_db.postgres_table;
```
## Dynamically adding new tables to replication {#dynamically-adding-table-to-replication}
After `MaterializedPostgreSQL` database is created, it does not automatically detect new tables in according PostgreSQL database. Such tables can be added manually:
``` sql
ATTACH TABLE postgres_database.new_table;
```
When specifying a specific list of tables in the database using the setting [materialized_postgresql_tables_list](../../operations/settings/settings.md#materialized-postgresql-tables-list), it will be updated to the current state, taking into account the tables which were added by the `ATTACH TABLE` query.
Warning: before version 21.13 adding table to replication left unremoved temprorary replication slot (named `{db_name}_ch_replication_slot_tmp`). If attaching tables in clickhouse version before 21.13, make sure to delete it manually (`SELECT pg_drop_replication_slot('{db_name}_ch_replication_slot_tmp')`). Otherwise disk usage will grow. Issue is fixed in 21.13.
## Dynamically removing tables from replication {#dynamically-removing-table-from-replication}
It is possible to remove specific tables from replication:
``` sql
DETACH TABLE postgres_database.table_to_remove;
```
## Settings {#settings}
- [materialized_postgresql_tables_list](../../operations/settings/settings.md#materialized-postgresql-tables-list)
- [materialized_postgresql_schema](../../operations/settings/settings.md#materialized-postgresql-schema)
- [materialized_postgresql_schema_list](../../operations/settings/settings.md#materialized-postgresql-schema-list)
- [materialized_postgresql_allow_automatic_update](../../operations/settings/settings.md#materialized-postgresql-allow-automatic-update)
- [materialized_postgresql_max_block_size](../../operations/settings/settings.md#materialized-postgresql-max-block-size)
- [materialized_postgresql_replication_slot](../../operations/settings/settings.md#materialized-postgresql-replication-slot)
- [materialized_postgresql_snapshot](../../operations/settings/settings.md#materialized-postgresql-snapshot)
``` sql
CREATE DATABASE database1
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password')
SETTINGS materialized_postgresql_tables_list = 'table1,table2,table3';
SELECT * FROM database1.table1;
```
The settings can be changed, if necessary, using a DDL query. But it is impossible to change the setting `materialized_postgresql_tables_list`. To update the list of tables in this setting use the `ATTACH TABLE` query.
``` sql
ALTER DATABASE postgres_database MODIFY SETTING materialized_postgresql_max_block_size = <new_size>;
```
## PostgreSQL schema {#schema}
PostgreSQL [schema](https://www.postgresql.org/docs/9.1/ddl-schemas.html) can be configured in 3 ways (starting from version 21.12).
@ -150,13 +138,63 @@ WHERE oid = 'postgres_table'::regclass;
!!! warning "Warning"
Replication of [**TOAST**](https://www.postgresql.org/docs/9.5/storage-toast.html) values is not supported. The default value for the data type will be used.
## Example of Use {#example-of-use}
## Settings {#settings}
1. materialized_postgresql_tables_list {#materialized-postgresql-tables-list}
Sets a comma-separated list of PostgreSQL database tables, which will be replicated via [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) database engine.
Default value: empty list — means whole PostgreSQL database will be replicated.
2. materialized_postgresql_schema {#materialized-postgresql-schema}
Default value: empty string. (Default schema is used)
3. materialized_postgresql_schema_list {#materialized-postgresql-schema-list}
Default value: empty list. (Default schema is used)
4. materialized_postgresql_allow_automatic_update {#materialized-postgresql-allow-automatic-update}
Allows reloading table in the background, when schema changes are detected. DDL queries on the PostgreSQL side are not replicated via ClickHouse [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) engine, because it is not allowed with PostgreSQL logical replication protocol, but the fact of DDL changes is detected transactionally. In this case, the default behaviour is to stop replicating those tables once DDL is detected. However, if this setting is enabled, then, instead of stopping the replication of those tables, they will be reloaded in the background via database snapshot without data losses and replication will continue for them.
Possible values:
- 0 — The table is not automatically updated in the background, when schema changes are detected.
- 1 — The table is automatically updated in the background, when schema changes are detected.
Default value: `0`.
5. materialized_postgresql_max_block_size {#materialized-postgresql-max-block-size}
Sets the number of rows collected in memory before flushing data into PostgreSQL database table.
Possible values:
- Positive integer.
Default value: `65536`.
6. materialized_postgresql_replication_slot {#materialized-postgresql-replication-slot}
A user-created replication slot. Must be used together with `materialized_postgresql_snapshot`.
7. materialized_postgresql_snapshot {#materialized-postgresql-snapshot}
A text string identifying a snapshot, from which [initial dump of PostgreSQL tables](../../engines/database-engines/materialized-postgresql.md) will be performed. Must be used together with `materialized_postgresql_replication_slot`.
``` sql
CREATE DATABASE postgresql_db
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password');
CREATE DATABASE database1
ENGINE = MaterializedPostgreSQL('postgres1:5432', 'postgres_database', 'postgres_user', 'postgres_password')
SETTINGS materialized_postgresql_tables_list = 'table1,table2,table3';
SELECT * FROM postgresql_db.postgres_table;
SELECT * FROM database1.table1;
```
The settings can be changed, if necessary, using a DDL query. But it is impossible to change the setting `materialized_postgresql_tables_list`. To update the list of tables in this setting use the `ATTACH TABLE` query.
``` sql
ALTER DATABASE postgres_database MODIFY SETTING materialized_postgresql_max_block_size = <new_size>;
```
## Notes {#notes}
@ -165,11 +203,11 @@ SELECT * FROM postgresql_db.postgres_table;
Logical Replication Slots which exist on the primary are not available on standby replicas.
So if there is a failover, new primary (the old physical standby) won’t be aware of any slots which were existing with old primary. This will lead to a broken replication from PostgreSQL.
A solution to this is to manage replication slots yourself and define a permanent replication slot (some information can be found [here](https://patroni.readthedocs.io/en/latest/SETTINGS.html)). You'll need to pass slot name via [materialized_postgresql_replication_slot](../../operations/settings/settings.md#materialized-postgresql-replication-slot) setting, and it has to be exported with `EXPORT SNAPSHOT` option. The snapshot identifier needs to be passed via [materialized_postgresql_snapshot](../../operations/settings/settings.md#materialized-postgresql-snapshot) setting.
A solution to this is to manage replication slots yourself and define a permanent replication slot (some information can be found [here](https://patroni.readthedocs.io/en/latest/SETTINGS.html)). You'll need to pass slot name via `materialized_postgresql_replication_slot` setting, and it has to be exported with `EXPORT SNAPSHOT` option. The snapshot identifier needs to be passed via `materialized_postgresql_snapshot` setting.
Please note that this should be used only if it is actually needed. If there is no real need for that or full understanding why, then it is better to allow the table engine to create and manage its own replication slot.
**Example (from [@bchrobot](https://github.com/bchrobot))**
**Example (from [@bchrobot](https://github.com/bchrobot))**
1. Configure replication slot in PostgreSQL.
@ -214,3 +252,23 @@ SETTINGS
```bash
kubectl exec acid-demo-cluster-0 -c postgres -- su postgres -c 'patronictl failover --candidate acid-demo-cluster-1 --force'
```
### Required permissions
1. [CREATE PUBLICATION](https://postgrespro.ru/docs/postgresql/14/sql-createpublication) -- create query privilege.
2. [CREATE_REPLICATION_SLOT](https://postgrespro.ru/docs/postgrespro/10/protocol-replication#PROTOCOL-REPLICATION-CREATE-SLOT) -- replication privelege.
3. [pg_drop_replication_slot](https://postgrespro.ru/docs/postgrespro/9.5/functions-admin#functions-replication) -- replication privilege or superuser.
4. [DROP PUBLICATION](https://postgrespro.ru/docs/postgresql/10/sql-droppublication) -- owner of publication (`username` in MaterializedPostgreSQL engine itself).
It is possible to avoid executing `2` and `3` commands and having those permissions. Use settings `materialized_postgresql_replication_slot` and `materialized_postgresql_snapshot`. But with much care.
Access to tables:
1. pg_publication
2. pg_replication_slots
3. pg_publication_tables

4
docs/en/engines/table-engines/integrations/materialized-postgresql.md
View File

@ -7,7 +7,7 @@ toc_title: MaterializedPostgreSQL
Creates ClickHouse table with an initial data dump of PostgreSQL table and starts replication process, i.e. executes background job to apply new changes as they happen on PostgreSQL table in the remote PostgreSQL database.
If more than one table is required, it is highly recommended to use the [MaterializedPostgreSQL](../../../engines/database-engines/materialized-postgresql.md) database engine instead of the table engine and use the [materialized_postgresql_tables_list](../../../operations/settings/settings.md#materialized-postgresql-tables-list) setting, which specifies the tables to be replicated. It will be much better in terms of CPU, fewer connections and fewer replication slots inside the remote PostgreSQL database.
If more than one table is required, it is highly recommended to use the [MaterializedPostgreSQL](../../../engines/database-engines/materialized-postgresql.md) database engine instead of the table engine and use the `materialized_postgresql_tables_list` setting, which specifies the tables to be replicated (will also be possible to add database `schema`). It will be much better in terms of CPU, fewer connections and fewer replication slots inside the remote PostgreSQL database.
## Creating a Table {#creating-a-table}
@ -38,7 +38,7 @@ PRIMARY KEY key;
- `_version` — Transaction counter. Type: [UInt64](../../../sql-reference/data-types/int-uint.md).
- `_sign` — Deletion mark. Type: [Int8](../../../sql-reference/data-types/int-uint.md). Possible values:
- `1` — Row is not deleted,
- `1` — Row is not deleted,
- `-1` — Row is deleted.
These columns do not need to be added when a table is created. They are always accessible in `SELECT` query.

25
docs/en/engines/table-engines/integrations/postgresql.md
View File

@ -36,6 +36,31 @@ The table structure can differ from the original PostgreSQL table structure:
- `schema` — Non-default table schema. Optional.
- `on conflict ...` — example: `ON CONFLICT DO NOTHING`. Optional. Note: adding this option will make insertion less efficient.
or via config (since version 21.11):
```
<named_collections>
<postgres1>
<host></host>
<port></port>
<username></username>
<password></password>
<table></table>
</postgres1>
<postgres2>
<host></host>
<port></port>
<username></username>
<password></password>
</postgres2>
</named_collections>
```
Some parameters can be overriden by key value arguments:
``` sql
SELECT * FROM postgresql(postgres1, schema='schema1', table='table1');
```
## Implementation Details {#implementation-details}
`SELECT` queries on PostgreSQL side run as `COPY (SELECT ...) TO STDOUT` inside read-only PostgreSQL transaction with commit after each `SELECT` query.

6
docs/en/interfaces/http.md
View File

@ -426,6 +426,9 @@ Next are the configuration methods for different `type`.
The following example defines the values of [max_threads](../operations/settings/settings.md#settings-max_threads) and `max_alter_threads` settings, then queries the system table to check whether these settings were set successfully.
!!! note "Warning"
To keep the default `handlers` such as` query`, `play`,` ping`, use the `<defaults/>` rule.
Example:
``` xml
@ -443,6 +446,7 @@ Example:
<query>SELECT name, value FROM system.settings WHERE name = {name_2:String}</query>
</handler>
</rule>
<defaults/>
</http_handlers>
```
@ -475,6 +479,7 @@ Example:
<query_param_name>query_param</query_param_name>
</handler>
</rule>
<defaults/>
</http_handlers>
```
@ -505,6 +510,7 @@ Return a message.
<response_content>Say Hi!</response_content>
</handler>
</rule>
<defaults/>
</http_handlers>
```

20
docs/en/interfaces/third-party/gui.md vendored
View File

@ -220,4 +220,24 @@ SeekTable is [free](https://www.seektable.com/help/cloud-pricing) for personal/i
[Chadmin](https://github.com/bun4uk/chadmin) is a simple UI where you can visualize your currently running queries on your ClickHouse cluster and info about them and kill them if you want.
### TABLUM.IO {#tablum_io}
[TABLUM.IO](https://tablum.io/) — an online query and analytics tool for ETL and visualization. It allows connecting to ClickHouse, query data via a versatile SQL console as well as to load data from static files and 3rd party services. TABLUM.IO can visualize data results as charts and tables.
Features:
- ETL: data loading from popular databases, local and remote files, API invocations.
- Versatile SQL console with syntax highlight and visual query builder.
- Data visualization as charts and tables.
- Data materialization and sub-queries.
- Data reporting to Slack, Telegram or email.
- Data pipelining via proprietary API.
- Data export in JSON, CSV, SQL, HTML formats.
- Web-based interface.
TABLUM.IO can be run as a self-hosted solution (as a docker image) or in the cloud.
License: [commercial](https://tablum.io/pricing) product with 3-month free period.
Try it out for free [in the cloud](https://tablum.io/try).
Learn more about the product at [TABLUM.IO](https://tablum.io/)
[Original article](https://clickhouse.com/docs/en/interfaces/third-party/gui/) <!--hide-->

2
docs/en/introduction/adopters.md
View File

@ -87,7 +87,7 @@ toc_title: Adopters
| <a href="https://kontur.ru" class="favicon">Kontur</a> | Software Development | Metrics | — | — | [Talk in Russian, November 2018](https://www.youtube.com/watch?v=U4u4Bd0FtrY) |
| <a href="https://www.kuaishou.com/" class="favicon">Kuaishou</a> | Video | — | — | — | [ClickHouse Meetup, October 2018](https://clickhouse.com/blog/en/2018/clickhouse-community-meetup-in-beijing-on-october-28-2018/) |
| <a href="https://www.kgk-global.com/en/" class="favicon">KGK Global</a> | Vehicle monitoring | — | — | — | [Press release, June 2021](https://zoom.cnews.ru/news/item/530921) |
| <a href="https://www.lbl.gov" class="favicon">Lawrence Berkeley National Laboratory</a> | Research | Traffic analysis | 1 server | 11.8 TiB | [Slides in English, April 2019](https://www.smitasin.com/presentations/2019-04-17_DOE-NSM.pdf) |
| <a href="https://www.lbl.gov" class="favicon">Lawrence Berkeley National Laboratory</a> | Research | Traffic analysis | 5 servers | 55 TiB | [Slides in English, April 2019](https://www.smitasin.com/presentations/2019-04-17_DOE-NSM.pdf) |
| <a href="https://lifestreet.com/" class="favicon">LifeStreet</a> | Ad network | Main product | 75 servers (3 replicas) | 5.27 PiB | [Blog post in Russian, February 2017](https://habr.com/en/post/322620/) |
| <a href="https://mcs.mail.ru/" class="favicon">Mail.ru Cloud Solutions</a> | Cloud services | Main product | — | — | [Article in Russian](https://mcs.mail.ru/help/db-create/clickhouse#) |
| <a href="https://maxilect.com/" class="favicon">MAXILECT</a> | Ad Tech, Blockchain, ML, AI | — | — | — | [Job advertisement, 2021](https://www.linkedin.com/feed/update/urn:li:activity:6780842017229430784/) |

6
docs/en/operations/external-authenticators/kerberos.md
View File

@ -14,11 +14,11 @@ To enable Kerberos, one should include `kerberos` section in `config.xml`. This
#### Parameters:
- `principal` - canonical service principal name that will be acquired and used when accepting security contexts.
- This parameter is optional, if omitted, the default principal will be used.
- This parameter is optional, if omitted, the default principal will be used.
- `realm` - a realm, that will be used to restrict authentication to only those requests whose initiator's realm matches it.
- This parameter is optional, if omitted, no additional filtering by realm will be applied.
- This parameter is optional, if omitted, no additional filtering by realm will be applied.
Example (goes into `config.xml`):
@ -75,7 +75,7 @@ In order to enable Kerberos authentication for the user, specify `kerberos` sect
Parameters:
- `realm` - a realm that will be used to restrict authentication to only those requests whose initiator's realm matches it.
- This parameter is optional, if omitted, no additional filtering by realm will be applied.
- This parameter is optional, if omitted, no additional filtering by realm will be applied.
Example (goes into `users.xml`):

46
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -435,26 +435,58 @@ Similar to `interserver_http_host`, except that this hostname can be used by oth
## interserver_http_credentials {#server-settings-interserver-http-credentials}
The username and password used to authenticate during [replication](../../engines/table-engines/mergetree-family/replication.md) with the Replicated\* engines. These credentials are used only for communication between replicas and are unrelated to credentials for ClickHouse clients. The server is checking these credentials for connecting replicas and use the same credentials when connecting to other replicas. So, these credentials should be set the same for all replicas in a cluster.
By default, the authentication is not used.
A username and a password used to connect to other servers during [replication](../../engines/table-engines/mergetree-family/replication.md). Also the server authenticates other replicas using these credentials. So, `interserver_http_credentials` must be the same for all replicas in a cluster.
By default, if `interserver_http_credentials` section is omitted, authentication is not used during replication.
!!! note "Note"
These credentials are common for replication through `HTTP` and `HTTPS`.
`interserver_http_credentials` settings do not relate to a ClickHouse client credentials [configuration](../../interfaces/cli.md#configuration_files).
This section contains the following parameters:
!!! note "Note"
These credentials are common for replication via `HTTP` and `HTTPS`.
- `user` — username.
- `password` — password.
The section contains the following parameters:
**Example**
- `user` — Username.
- `password` — Password.
- `allow_empty` — If `true`, then other replicas are allowed to connect without authentication even if credentials are set. If `false`, then connections without authentication are refused. Default value: `false`.
- `old` — Contains old `user` and `password` used during credential rotation. Several `old` sections can be specified.
**Credentials Rotation**
ClickHouse supports dynamic interserver credentials rotation without stopping all replicas at the same time to update their configuration. Credentials can be changed in several steps.
To enable authentication, set `interserver_http_credentials.allow_empty` to `true` and add credentials. This allows connections with authentication and without it.
``` xml
<interserver_http_credentials>
<user>admin</user>
<password>111</password>
<allow_empty>true</allow_empty>
</interserver_http_credentials>
```
After configuring all replicas set `allow_empty` to `false` or remove this setting. It makes authentication with new credentials mandatory.
To change existing credentials, move the username and the password to `interserver_http_credentials.old` section and update `user` and `password` with new values. At this point the server uses new credentials to connect to other replicas and accepts connections with either new or old credentials.
``` xml
<interserver_http_credentials>
<user>admin</user>
<password>222</password>
<old>
<user>admin</user>
<password>111</password>
</old>
<old>
<user>temp</user>
<password>000</password>
</old>
</interserver_http_credentials>
```
When new credentials are applied to all replicas, old credentials may be removed.
## keep_alive_timeout {#keep-alive-timeout}
The number of seconds that ClickHouse waits for incoming requests before closing the connection. Defaults to 10 seconds.

53
docs/en/operations/settings/settings.md
View File

@ -1469,7 +1469,7 @@ Possible values:
Default value: `1`.
**See Also**
**See Also**
- [min_count_to_compile_aggregate_expression](#min_count_to_compile_aggregate_expression)
@ -2095,7 +2095,7 @@ Possible values:
- 0 — Optimization disabled.
- 1 — Optimization enabled.
Default value: `1`.
See also:
@ -3682,49 +3682,6 @@ Possible values:
Default value: `0`.
## materialized_postgresql_max_block_size {#materialized-postgresql-max-block-size}
Sets the number of rows collected in memory before flushing data into PostgreSQL database table.
Possible values:
- Positive integer.
Default value: `65536`.
## materialized_postgresql_tables_list {#materialized-postgresql-tables-list}
Sets a comma-separated list of PostgreSQL database tables, which will be replicated via [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) database engine.
Default value: empty list — means whole PostgreSQL database will be replicated.
## materialized_postgresql_schema {#materialized-postgresql-schema}
Default value: empty string. (Default schema is used)
## materialized_postgresql_schema_list {#materialized-postgresql-schema-list}
Default value: empty list. (Default schema is used)
## materialized_postgresql_allow_automatic_update {#materialized-postgresql-allow-automatic-update}
Allows reloading table in the background, when schema changes are detected. DDL queries on the PostgreSQL side are not replicated via ClickHouse [MaterializedPostgreSQL](../../engines/database-engines/materialized-postgresql.md) engine, because it is not allowed with PostgreSQL logical replication protocol, but the fact of DDL changes is detected transactionally. In this case, the default behaviour is to stop replicating those tables once DDL is detected. However, if this setting is enabled, then, instead of stopping the replication of those tables, they will be reloaded in the background via database snapshot without data losses and replication will continue for them.
Possible values:
- 0 — The table is not automatically updated in the background, when schema changes are detected.
- 1 — The table is automatically updated in the background, when schema changes are detected.
Default value: `0`.
## materialized_postgresql_replication_slot {#materialized-postgresql-replication-slot}
A user-created replication slot. Must be used together with [materialized_postgresql_snapshot](#materialized-postgresql-snapshot).
## materialized_postgresql_snapshot {#materialized-postgresql-snapshot}
A text string identifying a snapshot, from which [initial dump of PostgreSQL tables](../../engines/database-engines/materialized-postgresql.md) will be performed. Must be used together with [materialized_postgresql_replication_slot](#materialized-postgresql-replication-slot).
## allow_experimental_projection_optimization {#allow-experimental-projection-optimization}
Enables or disables [projection](../../engines/table-engines/mergetree-family/mergetree.md#projections) optimization when processing `SELECT` queries.
@ -3993,8 +3950,8 @@ If [wait_for_async_insert](#wait-for-async-insert) is enabled, every client will
Possible values:
- 0 — Insertions are made synchronously, one after another.
- 1 — Multiple asynchronous insertions enabled.
- 0 — Insertions are made synchronously, one after another.
- 1 — Multiple asynchronous insertions enabled.
Default value: `0`.
@ -4066,7 +4023,7 @@ Default value: `0`.
## alter_partition_verbose_result {#alter-partition-verbose-result}
Enables or disables the display of information about the parts to which the manipulation operations with partitions and parts have been successfully applied.
Enables or disables the display of information about the parts to which the manipulation operations with partitions and parts have been successfully applied.
Applicable to [ATTACH PARTITION|PART](../../sql-reference/statements/alter/partition.md#alter_attach-partition) and to [FREEZE PARTITION](../../sql-reference/statements/alter/partition.md#alter_freeze-partition).
Possible values:

2
docs/en/sql-reference/statements/create/role.md
View File

@ -31,7 +31,7 @@ CREATE ROLE accountant;
GRANT SELECT ON db.* TO accountant;
```
This sequence of queries creates the role `accountant` that has the privilege of reading data from the `accounting` database.
This sequence of queries creates the role `accountant` that has the privilege of reading data from the `db` database.
Assigning the role to the user `mira`:

58
docs/en/sql-reference/table-functions/hdfsCluster.md Normal file
View File

@ -0,0 +1,58 @@
---
toc_priority: 55
toc_title: hdfsCluster
---
# hdfsCluster Table Function {#hdfsCluster-table-function}
Allows processing files from HDFS in parallel from many nodes in a specified cluster. On initiator it creates a connection to all nodes in the cluster, discloses asterics in HDFS file path, and dispatches each file dynamically. On the worker node it asks the initiator about the next task to process and processes it. This is repeated until all tasks are finished.
**Syntax**
``` sql
hdfsCluster(cluster_name, URI, format, structure)
```
**Arguments**
- `cluster_name` — Name of a cluster that is used to build a set of addresses and connection parameters to remote and local servers.
- `URI` — URI to a file or a bunch of files. Supports following wildcards in readonly mode: `*`, `?`, `{'abc','def'}` and `{N..M}` where `N`, `M` — numbers, `abc`, `def` — strings. For more information see [Wildcards In Path](../../engines/table-engines/integrations/s3.md#wildcards-in-path).
- `format` — The [format](../../interfaces/formats.md#formats) of the file.
- `structure` — Structure of the table. Format `'column1_name column1_type, column2_name column2_type, ...'`.
**Returned value**
A table with the specified structure for reading data in the specified file.
**Examples**
1. Suppose that we have a ClickHouse cluster named `cluster_simple`, and several files with following URIs on HDFS:
- ‘hdfs://hdfs1:9000/some_dir/some_file_1’
- ‘hdfs://hdfs1:9000/some_dir/some_file_2’
- ‘hdfs://hdfs1:9000/some_dir/some_file_3’
- ‘hdfs://hdfs1:9000/another_dir/some_file_1’
- ‘hdfs://hdfs1:9000/another_dir/some_file_2’
- ‘hdfs://hdfs1:9000/another_dir/some_file_3’
2. Query the amount of rows in these files:
``` sql
SELECT count(*)
FROM hdfsCluster('cluster_simple', 'hdfs://hdfs1:9000/{some,another}_dir/some_file_{1..3}', 'TSV', 'name String, value UInt32')
```
3. Query the amount of rows in all files of these two directories:
``` sql
SELECT count(*)
FROM hdfsCluster('cluster_simple', 'hdfs://hdfs1:9000/{some,another}_dir/*', 'TSV', 'name String, value UInt32')
```
!!! warning "Warning"
If your listing of files contains number ranges with leading zeros, use the construction with braces for each digit separately or use `?`.
**See Also**
- [HDFS engine](../../engines/table-engines/integrations/hdfs.md)
- [HDFS table function](../../sql-reference/table-functions/hdfs.md)

10
docs/ja/getting-started/example-datasets/ontime.md
View File

@ -15,13 +15,7 @@ toc_title: OnTime
データのダウンロード:
``` bash
for s in `seq 1987 2018`
do
for m in `seq 1 12`
do
wget https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_${s}_${m}.zip
done
done
wget --no-check-certificate --continue https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{1987..2021}_{1..12}.zip
```
（https://github.com/Percona-Lab/ontime-airline-performance/blob/master/download.sh より)
@ -40,7 +34,7 @@ CREATE TABLE `ontime`
`Reporting_Airline` String,
`DOT_ID_Reporting_Airline` Int32,
`IATA_CODE_Reporting_Airline` String,
`Tail_Number` Int32,
`Tail_Number` String,
`Flight_Number_Reporting_Airline` String,
`OriginAirportID` Int32,
`OriginAirportSeqID` Int32,

10
docs/ru/getting-started/example-datasets/ontime.md
View File

@ -15,13 +15,7 @@ toc_title: OnTime
Скачивание данных (из `https://github.com/Percona-Lab/ontime-airline-performance/blob/master/download.sh`):
``` bash
for s in `seq 1987 2018`
do
for m in `seq 1 12`
do
wget https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_${s}_${m}.zip
done
done
wget --no-check-certificate --continue https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{1987..2021}_{1..12}.zip
```
Создание таблицы:
@ -38,7 +32,7 @@ CREATE TABLE `ontime`
`Reporting_Airline` String,
`DOT_ID_Reporting_Airline` Int32,
`IATA_CODE_Reporting_Airline` String,
`Tail_Number` Int32,
`Tail_Number` String,
`Flight_Number_Reporting_Airline` String,
`OriginAirportID` Int32,
`OriginAirportSeqID` Int32,

4
docs/ru/interfaces/cli.md
View File

@ -114,7 +114,7 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe
Параметры в конфигурационных файлах переопределяют значения по умолчанию.
### Параметры командной строки {#parametry-komandnoi-stroki}
### Параметры командной строки {#command-line-options}
- `--host, -h` — имя сервера, по умолчанию — ‘localhost’. Вы можете использовать как имя, так и IPv4 или IPv6 адрес.
- `--port` — порт для подключения, по умолчанию — 9000. Обратите внимание: для HTTP-интерфейса и нативного интерфейса используются разные порты.
@ -136,7 +136,7 @@ $ clickhouse-client --param_tbl="numbers" --param_db="system" --param_col="numbe
Начиная с версии 20.5, в `clickhouse-client` есть автоматическая подсветка синтаксиса (включена всегда).
### Конфигурационные файлы {#konfiguratsionnye-faily}
### Конфигурационные файлы {#configuration_files}
`clickhouse—client` использует первый существующий файл из:

6
docs/ru/interfaces/http.md
View File

@ -424,6 +424,9 @@ $ curl -v 'http://localhost:8123/predefined_query'
В следующем примере определяются настройки [max_threads](../operations/settings/settings.md#settings-max_threads) и `max_alter_threads`, а затем запрашивается системная таблица, чтобы проверить, были ли эти параметры успешно установлены.
!!! note "Предупреждение"
Чтобы сохранить стандартные `handlers` такие как `query`, `play`, `ping`, используйте правило `<defaults/>`.
Пример:
``` xml
@ -441,6 +444,7 @@ $ curl -v 'http://localhost:8123/predefined_query'
<query>SELECT name, value FROM system.settings WHERE name = {name_2:String}</query>
</handler>
</rule>
<defaults/>
</http_handlers>
```
@ -473,6 +477,7 @@ ClickHouse извлекает и выполняет значение, соотв
<query_param_name>query_param</query_param_name>
</handler>
</rule>
<defaults/>
</http_handlers>
```
@ -503,6 +508,7 @@ max_alter_threads 2
<response_content>Say Hi!</response_content>
</handler>
</rule>
<defaults/>
</http_handlers>
```

21
docs/ru/interfaces/third-party/gui.md vendored
View File

@ -227,4 +227,25 @@ SeekTable [бесплатен](https://www.seektable.com/help/cloud-pricing) д
[Chadmin](https://github.com/bun4uk/chadmin) — простой графический интерфейс для визуализации запущенных запросов на вашем кластере ClickHouse. Он отображает информацию о запросах и дает возможность их завершать.
### TABLUM.IO {#tablum_io}
[TABLUM.IO](https://tablum.io/) — онлайн инструмент для загрузки и визуализации данных. Позволяет подключаться к БД ClickHouse, работать с базами и таблицами через многофункциональную SQL консоль, загружать данные из таблиц, объединять их с данными из других источников (файлов, сторонних сервисов) и визуализировать результаты в виде таблиц и графиков.
Основные возможности:
- Многофункциональный ETL: загрузка данных из популярных баз данных, локальных и удаленных файлов, загрузка результатов вызова REST API.
- Универсальная SQL консоль с подсветкой синтаксиса и визуальным генератором SQL запросов.
- Визуализация загруженных данных в виде графиков и таблиц.
- Материализация данных и подзапросы к загруженным данным.
- Отправка результатов визуализации в Slack, Telegram или на email.
- Организация потоков данных (data pipeline) через собственный API.
- Экспорт данных в форматах JSON, CSV, SQL, HTML.
- Веб-интерфейс.
Поддерживается установка TABLUM.IO на собственный сервер (в виде Docker образа) или работа с сервисом в облаке.
Лицензия: [коммерческий](https://tablum.io/pricing) продукт с периодом бесплатного тестирования 3 месяца.
Протестировать TABLUM.IO без разворачивания на собственном сервере можно [здесь](https://tablum.io/try).
Подробно о продукте смотрите на [TABLUM.IO](https://tablum.io/)
[Original article](https://clickhouse.com/docs/en/interfaces/third-party/gui/) <!--hide-->

40
docs/ru/operations/server-configuration-parameters/settings.md
View File

@ -436,26 +436,58 @@ ClickHouse проверяет условия для `min_part_size` и `min_part
## interserver_http_credentials {#server-settings-interserver-http-credentials}
Имя пользователя и пароль, использующиеся для аутентификации при [репликации](../../operations/server-configuration-parameters/settings.md) движками Replicated\*. Это имя пользователя и пароль используются только для взаимодействия между репликами кластера и никак не связаны с аутентификацией клиентов ClickHouse. Сервер проверяет совпадение имени и пароля для соединяющихся с ним реплик, а также использует это же имя и пароль для соединения с другими репликами. Соответственно, эти имя и пароль должны быть прописаны одинаковыми для всех реплик кластера.
По умолчанию аутентификация не используется.
Имя пользователя и пароль, использующиеся для подключения к другим серверам при [репликации](../../engines/table-engines/mergetree-family/replication.md) движками Replicated\*. Сервер использует эти же учетные данные при аутентификации других реплик. Поэтому настройки `interserver_http_credentials` должны быть заданы одинаковыми для всех реплик кластера.
По умолчанию, если секция `interserver_http_credentials` не задана в конфигурации, аутентификация при репликации не используется.
!!! note "Примечание"
Эти учетные данные являются общими для обмена данными по протоколам `HTTP` и `HTTPS`.
Настройки `interserver_http_credentials` не относятся к [конфигурации](../../interfaces/cli.md#configuration_files) учетных данных клиента ClickHouse.
!!! note "Примечание"
Учетные данные в `interserver_http_credentials` являются общими для репликации по `HTTP` и `HTTPS`.
Раздел содержит следующие параметры:
- `user` — имя пользователя.
- `password` — пароль.
- `allow_empty` — если `true`, то другие реплики могут подключаться без аутентификации, даже если учетные данные заданы. Если `false`, то подключение без аутентификации не допускается. Значение по умолчанию: `false`.
- `old` — секция содержит старые значения `user` и `password`, которые используются в процессе изменения учетных данных. Можно указывать несколько секций `old`.
**Пример конфигурации**
**Изменение учетных данных**
ClickHouse поддерживает динамическое изменение учетных данных. При этом не требуется одновременно останавливать все реплики, чтобы обновить конфигурацию. Изменение учетных данных выполняется за несколько шагов.
Чтобы включить аутентификацию, установите `interserver_http_credentials.allow_empty` в значение `true` и задайте учетные данные. С такой конфигурацией разрешены подключения как с аутентификацией, так и без нее.
``` xml
<interserver_http_credentials>
<user>admin</user>
<password>111</password>
<allow_empty>true</allow_empty>
</interserver_http_credentials>
```
После конфигурации всех реплик установите `allow_empty` в значение `false` или удалите эту настройку. Это сделает аутентификацию с новыми учетными данными обязательной.
Чтобы изменить учетные данные, перенесите имя пользователя и пароль в секцию `interserver_http_credentials.old` и укажите новые значения для `user` и `password`. Сервер будет использовать новые учетные данные при подключении к другим репликам и при этом будет разрешать подключения как с новыми, так и со старыми учетными данными.
``` xml
<interserver_http_credentials>
<user>admin</user>
<password>222</password>
<old>
<user>admin</user>
<password>111</password>
</old>
<old>
<user>temp</user>
<password>000</password>
</old>
</interserver_http_credentials>
```
Когда новые учетные данные обновятся на всех репликах, старые учетные данные можно удалить из конфигурации.
## keep_alive_timeout {#keep-alive-timeout}
Время в секундах, в течение которого ClickHouse ожидает входящих запросов прежде чем закрыть соединение. Значение по умолчанию: 10 секунд.

12
docs/zh/getting-started/example-datasets/ontime.md
View File

@ -15,17 +15,9 @@ toc_title: OnTime
下载数据：
``` bash
for s in `seq 1987 2018`
do
for m in `seq 1 12`
do
wget https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_${s}_${m}.zip
done
done
wget --no-check-certificate --continue https://transtats.bts.gov/PREZIP/On_Time_Reporting_Carrier_On_Time_Performance_1987_present_{1987..2021}_{1..12}.zip
```
(参考 https://github.com/Percona-Lab/ontime-airline-performance/blob/master/download.sh )
创建表结构：
``` sql
@ -40,7 +32,7 @@ CREATE TABLE `ontime`
`Reporting_Airline` String,
`DOT_ID_Reporting_Airline` Int32,
`IATA_CODE_Reporting_Airline` String,
`Tail_Number` Int32,
`Tail_Number` String,
`Flight_Number_Reporting_Airline` String,
`OriginAirportID` Int32,
`OriginAirportSeqID` Int32,

43
docs/zh/operations/system-tables/databases.md
View File

@ -1,12 +1,39 @@
---
machine_translated: true
machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3
---
# system.databases {#system-databases}
# 系统。数据库 {#system-databases}
包含当前用户可用的数据库的相关信息。
此表包含一个名为"字符串"的列 ‘name’ – the name of a database.
列:
服务器知道的每个数据库在表中都有相应的条目。
- `name` ([String](../../sql-reference/data-types/string.md)) — 数据库的名称。
- `engine` ([String](../../sql-reference/data-types/string.md)) — [数据库的引擎](../../engines/database-engines/index.md)。
- `data_path` ([String](../../sql-reference/data-types/string.md)) — 数据的路径。
- `metadata_path` ([String](../../sql-reference/data-types/enum.md)) — 元数据的路径。
- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — 数据库的 UUID。
- `comment` ([String](../../sql-reference/data-types/enum.md)) — 数据库的注释。
该系统表用于实现 `SHOW DATABASES` 查询。
这个系统表的 `name` 列被用于实现 `SHOW DATABASES` 查询。
**示例**
创建一个数据库。
``` sql
CREATE DATABASE test;
```
查询此用户所有可用的数据库。
``` sql
SELECT * FROM system.databases;
```
``` text
┌─name───────────────┬─engine─┬─data_path──────────────────┬─metadata_path───────────────────────────────────────────────────────┬─uuid─────────────────────────────────┬─comment─┐
│ INFORMATION_SCHEMA │ Memory │ /var/lib/clickhouse/ │ │ 00000000-0000-0000-0000-000000000000 │ │
│ default │ Atomic │ /var/lib/clickhouse/store/ │ /var/lib/clickhouse/store/d31/d317b4bd-3595-4386-81ee-c2334694128a/ │ 24363899-31d7-42a0-a436-389931d752a0 │ │
│ information_schema │ Memory │ /var/lib/clickhouse/ │ │ 00000000-0000-0000-0000-000000000000 │ │
│ system │ Atomic │ /var/lib/clickhouse/store/ │ /var/lib/clickhouse/store/1d1/1d1c869d-e465-4b1b-a51f-be033436ebf9/ │ 03e9f3d1-cc88-4a49-83e9-f3d1cc881a49 │ │
└────────────────────┴────────┴────────────────────────────┴─────────────────────────────────────────────────────────────────────┴──────────────────────────────────────┴─────────┘
```
[原文](https://clickhouse.com/docs/zh/operations/system-tables/databases) <!--hide-->

40
docs/zh/operations/system-tables/disks.md
View File

@ -1,31 +1,27 @@
---
machine_translated: true
machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3
---
# system.disks {#system_tables-disks}
# 系统。磁盘 {#system_tables-disks}
包含有关在定义的磁盘信息 [服务器配置](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure).
包含在 [服务器配置](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure) 中定义的磁盘信息.
列:
- `name` ([字符串](../../sql-reference/data-types/string.md)) — Name of a disk in the server configuration.
- `path` ([字符串](../../sql-reference/data-types/string.md)) — Path to the mount point in the file system.
- `free_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Free space on disk in bytes.
- `total_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Disk volume in bytes.
- `keep_free_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Amount of disk space that should stay free on disk in bytes. Defined in the `keep_free_space_bytes` 磁盘配置参数。
- `name` ([字符串](../../sql-reference/data-types/string.md)) — 服务器配置中的磁盘名称.
- `path` ([字符串](../../sql-reference/data-types/string.md)) — 文件系统中挂载点的路径.
- `free_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — 磁盘上的可用空间，以字节为单位.
- `total_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — 磁盘容量，以字节为单位。
- `keep_free_space` ([UInt64](../../sql-reference/data-types/int-uint.md)) — 在磁盘上应保持空闲的磁盘空间的数量，以字节为单位。在磁盘配置的 `keep_free_space_bytes` 参数中定义。
## 系统。storage_policies {#system_tables-storage_policies}
**示例**
包含有关存储策略和卷中定义的信息 [服务器配置](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure).
```sql
:) SELECT * FROM system.disks;
```
列:
```text
┌─name────┬─path─────────────────┬───free_space─┬──total_space─┬─keep_free_space─┐
│ default │ /var/lib/clickhouse/ │ 276392587264 │ 490652508160 │ 0 │
└─────────┴──────────────────────┴──────────────┴──────────────┴─────────────────┘
- `policy_name` ([字符串](../../sql-reference/data-types/string.md)) — Name of the storage policy.
- `volume_name` ([字符串](../../sql-reference/data-types/string.md)) — Volume name defined in the storage policy.
- `volume_priority` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Volume order number in the configuration.
- `disks` ([数组（字符串)](../../sql-reference/data-types/array.md)) — Disk names, defined in the storage policy.
- `max_data_part_size` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Maximum size of a data part that can be stored on volume disks (0 — no limit).
- `move_factor` ([Float64](../../sql-reference/data-types/float.md)) — Ratio of free disk space. When the ratio exceeds the value of configuration parameter, ClickHouse start to move data to the next volume in order.
1 rows in set. Elapsed: 0.001 sec.
```
如果存储策略包含多个卷，则每个卷的信息将存储在表的单独行中。
[原文](https://clickhouse.com/docs/zh/operations/system-tables/disks) <!--hide-->

27
docs/zh/operations/system-tables/one.md
View File

@ -1,12 +1,23 @@
---
machine_translated: true
machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3
---
# system.one {#system-one}
# 系统。一 {#system-one}
此表包含一行只有一个值为 0 的 `dummy` UInt8 列的数据。
此表包含一行，其中包含一行 `dummy` UInt8列包含值0。
如果 `SELECT` 查询没有指定 `FROM` 子句，就会使用这个表来查询。
如果使用此表 `SELECT` 查询不指定 `FROM` 条款
这个表类似于其他数据库管理系统(DMBS)中的 `DUAL` 表。
这类似于 `DUAL` 表在其他Dbms中找到。
**示例**
```sql
:) SELECT * FROM system.one LIMIT 10;
```
```text
┌─dummy─┐
│ 0 │
└───────┘
1 rows in set. Elapsed: 0.001 sec.
```
[原文](https://clickhouse.com/docs/zh/operations/system-tables/one) <!--hide-->

26
docs/zh/operations/system-tables/storage_policies.md
View File

@ -1,19 +1,17 @@
---
machine_translated: true
machine_translated_rev: 5decc73b5dc60054f19087d3690c4eb99446a6c3
---
# system.storage_policies {#system_tables-storage_policies}
# 系统。storage_policies {#system_tables-storage_policies}
包含有关存储策略和卷中定义的信息 [服务器配置](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure).
包含有关 [服务器配置](../../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes_configure) 中定义的存储策略和卷信息。
列:
- `policy_name` ([字符串](../../sql-reference/data-types/string.md)) — Name of the storage policy.
- `volume_name` ([字符串](../../sql-reference/data-types/string.md)) — Volume name defined in the storage policy.
- `volume_priority` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Volume order number in the configuration.
- `disks` ([数组（字符串)](../../sql-reference/data-types/array.md)) — Disk names, defined in the storage policy.
- `max_data_part_size` ([UInt64](../../sql-reference/data-types/int-uint.md)) — Maximum size of a data part that can be stored on volume disks (0 — no limit).
- `move_factor` ([Float64](../../sql-reference/data-types/float.md)) — Ratio of free disk space. When the ratio exceeds the value of configuration parameter, ClickHouse start to move data to the next volume in order.
- `policy_name` ([String](../../sql-reference/data-types/string.md)) — 存储策略的名称。
- `volume_name` ([String](../../sql-reference/data-types/string.md)) — 存储策略中定义的卷名称。
- `volume_priority` ([UInt64](../../sql-reference/data-types/int-uint.md)) — 配置中的卷顺序号，数据根据这个优先级填充卷，比如插入和合并期间的数据将被写入优先级较低的卷 (还需考虑其他规则: TTL, `max_data_part_size`, `move_factor`)。
- `disks` ([Array(String)](../../sql-reference/data-types/array.md)) — 存储策略中定义的磁盘名。
- `max_data_part_size` ([UInt64](../../sql-reference/data-types/int-uint.md)) — 可以存储在卷磁盘上数据部分的最大大小 (0 - 不限制)。
- `move_factor` ([Float64](../../sql-reference/data-types/float.md)) — 磁盘空闲的比率。当比率超过配置的值，ClickHouse 将把数据向下一个卷移动。
- `prefer_not_to_merge` ([UInt8](../../sql-reference/data-types/int-uint.md)) — 设置中 `prefer_not_to_merge` 的值. 当这个设置启用时，不允许在此卷上合并数据。这将允许控制 ClickHouse 如何与运行速度较慢的磁盘一起工作。
如果存储策略包含多个卷，则每个卷的信息将存储在表的单独行中。
如果存储策略包含多个卷，则每个卷的信息将在表中作为单独一行存储。
[原文](https://clickhouse.com/docs/zh/operations/system-tables/storage_policies) <!--hide-->

2
programs/install/Install.cpp
View File

@ -20,7 +20,7 @@
#include <Common/Config/ConfigProcessor.h>
#include <Common/OpenSSLHelpers.h>
#include <Common/hex.h>
#include <base/getResource.h>
#include <Common/getResource.h>
#include <base/sleep.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/WriteBufferFromFileDescriptor.h>

2
programs/local/LocalServer.cpp
View File

@ -792,9 +792,9 @@ void LocalServer::processOptions(const OptionsDescription &, const CommandLineOp
int mainEntryClickHouseLocal(int argc, char ** argv)
{
DB::LocalServer app;
try
{
DB::LocalServer app;
app.init(argc, argv);
return app.run();
}

2
programs/obfuscator/Obfuscator.cpp
View File

@ -30,7 +30,7 @@
#include <Processors/Executors/PushingPipelineExecutor.h>
#include <Core/Block.h>
#include <base/StringRef.h>
#include <base/DateLUT.h>
#include <Common/DateLUT.h>
#include <base/bit_cast.h>
#include <IO/ReadBufferFromFileDescriptor.h>
#include <IO/WriteBufferFromFileDescriptor.h>

11
programs/odbc-bridge/ODBCBlockOutputStream.cpp
View File

@ -1,13 +1,8 @@
#include "ODBCBlockOutputStream.h"
#include <Common/hex.h>
#include <base/logger_useful.h>
#include <Core/Field.h>
#include <base/LocalDate.h>
#include <base/LocalDateTime.h>
#include "getIdentifierQuote.h"
#include <IO/WriteHelpers.h>
#include <IO/Operators.h>
#include <IO/WriteBufferFromString.h>
#include <Interpreters/Context.h>
#include <Processors/Formats/IOutputFormat.h>
#include <Parsers/getInsertQuery.h>
@ -45,7 +40,7 @@ void ODBCSink::consume(Chunk chunk)
std::string query = getInsertQuery(db_name, table_name, block.getColumnsWithTypeAndName(), quoting) + values_buf.str();
execute<void>(connection_holder,
[&](nanodbc::connection & connection) { execute(connection, query); });
[&](nanodbc::connection & connection) { execute(connection, query); });
}
}

16
src/Access/AccessControl.cpp
View File

@ -30,6 +30,7 @@ namespace ErrorCodes
{
extern const int UNKNOWN_ELEMENT_IN_CONFIG;
extern const int UNKNOWN_SETTING;
extern const int AUTHENTICATION_FAILED;
}
@ -401,9 +402,20 @@ void AccessControl::addStoragesFromMainConfig(
}
UUID AccessControl::login(const Credentials & credentials, const Poco::Net::IPAddress & address) const
UUID AccessControl::authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const
{
return MultipleAccessStorage::login(credentials, address, *external_authenticators);
try
{
return MultipleAccessStorage::authenticate(credentials, address, *external_authenticators);
}
catch (...)
{
tryLogCurrentException(getLogger(), "from: " + address.toString() + ", user: " + credentials.getUserName() + ": Authentication failed");
/// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons,
/// only the log will show the exact reason.
throw Exception(credentials.getUserName() + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED);
}
}
void AccessControl::setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config)

2
src/Access/AccessControl.h
View File

@ -113,7 +113,7 @@ public:
bool isSettingNameAllowed(const std::string_view & name) const;
void checkSettingNameIsAllowed(const std::string_view & name) const;
UUID login(const Credentials & credentials, const Poco::Net::IPAddress & address) const;
UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address) const;
void setExternalAuthenticatorsConfig(const Poco::Util::AbstractConfiguration & config);
std::shared_ptr<const ContextAccess> getContextAccess(

97
src/Access/DiskAccessStorage.cpp
View File

@ -426,19 +426,24 @@ std::vector<UUID> DiskAccessStorage::findAllImpl(AccessEntityType type) const
return res;
}
bool DiskAccessStorage::existsImpl(const UUID & id) const
bool DiskAccessStorage::exists(const UUID & id) const
{
std::lock_guard lock{mutex};
return entries_by_id.count(id);
}
AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id) const
AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
{
std::lock_guard lock{mutex};
auto it = entries_by_id.find(id);
if (it == entries_by_id.end())
throwNotFound(id);
{
if (throw_if_not_exists)
throwNotFound(id);
else
return nullptr;
}
const auto & entry = it->second;
if (!entry.entity)
@ -447,43 +452,56 @@ AccessEntityPtr DiskAccessStorage::readImpl(const UUID & id) const
}
String DiskAccessStorage::readNameImpl(const UUID & id) const
std::optional<String> DiskAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
{
std::lock_guard lock{mutex};
auto it = entries_by_id.find(id);
if (it == entries_by_id.end())
throwNotFound(id);
return String{it->second.name};
{
if (throw_if_not_exists)
throwNotFound(id);
else
return std::nullopt;
}
return it->second.name;
}
bool DiskAccessStorage::canInsertImpl(const AccessEntityPtr &) const
{
return !readonly;
}
UUID DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists)
std::optional<UUID> DiskAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
UUID id = generateRandomID();
std::lock_guard lock{mutex};
insertNoLock(id, new_entity, replace_if_exists, notifications);
return id;
if (insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, notifications))
return id;
return std::nullopt;
}
void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications)
bool DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, Notifications & notifications)
{
const String & name = new_entity->getName();
AccessEntityType type = new_entity->getType();
/// Check that we can insert.
if (readonly)
throwReadonlyCannotInsert(type, name);
/// Check that we can insert.
auto & entries_by_name = entries_by_name_and_type[static_cast<size_t>(type)];
auto it_by_name = entries_by_name.find(name);
bool name_collision = (it_by_name != entries_by_name.end());
if (name_collision && !replace_if_exists)
{
if (throw_if_exists)
throwNameCollisionCannotInsert(type, name);
else
return false;
}
auto it_by_id = entries_by_id.find(id);
if (it_by_id != entries_by_id.end())
{
@ -491,18 +509,11 @@ void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne
throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName());
}
auto & entries_by_name = entries_by_name_and_type[static_cast<size_t>(type)];
auto it_by_name = entries_by_name.find(name);
bool name_collision = (it_by_name != entries_by_name.end());
if (name_collision && !replace_if_exists)
throwNameCollisionCannotInsert(type, name);
scheduleWriteLists(type);
writeAccessEntityToDisk(id, *new_entity);
if (name_collision && replace_if_exists)
removeNoLock(it_by_name->second->id, notifications);
removeNoLock(it_by_name->second->id, /* throw_if_not_exists = */ false, notifications);
/// Do insertion.
auto & entry = entries_by_id[id];
@ -512,24 +523,30 @@ void DiskAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & ne
entry.entity = new_entity;
entries_by_name[entry.name] = &entry;
prepareNotifications(id, entry, false, notifications);
return true;
}
void DiskAccessStorage::removeImpl(const UUID & id)
bool DiskAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists)
{
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
std::lock_guard lock{mutex};
removeNoLock(id, notifications);
return removeNoLock(id, throw_if_not_exists, notifications);
}
void DiskAccessStorage::removeNoLock(const UUID & id, Notifications & notifications)
bool DiskAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists, Notifications & notifications)
{
auto it = entries_by_id.find(id);
if (it == entries_by_id.end())
throwNotFound(id);
{
if (throw_if_not_exists)
throwNotFound(id);
else
return false;
}
Entry & entry = it->second;
AccessEntityType type = entry.type;
@ -545,28 +562,35 @@ void DiskAccessStorage::removeNoLock(const UUID & id, Notifications & notificati
auto & entries_by_name = entries_by_name_and_type[static_cast<size_t>(type)];
entries_by_name.erase(entry.name);
entries_by_id.erase(it);
return true;
}
void DiskAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func)
bool DiskAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists)
{
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
std::lock_guard lock{mutex};
updateNoLock(id, update_func, notifications);
return updateNoLock(id, update_func, throw_if_not_exists, notifications);
}
void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications)
bool DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, Notifications & notifications)
{
auto it = entries_by_id.find(id);
if (it == entries_by_id.end())
throwNotFound(id);
{
if (throw_if_not_exists)
throwNotFound(id);
else
return false;
}
Entry & entry = it->second;
if (readonly)
throwReadonlyCannotUpdate(entry.type, entry.name);
if (!entry.entity)
entry.entity = readAccessEntityFromDisk(id);
auto old_entity = entry.entity;
@ -576,7 +600,7 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_
throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());
if (*new_entity == *old_entity)
return;
return true;
const String & new_name = new_entity->getName();
const String & old_name = old_entity->getName();
@ -602,6 +626,7 @@ void DiskAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_
}
prepareNotifications(id, entry, false, notifications);
return true;
}
@ -675,7 +700,7 @@ scope_guard DiskAccessStorage::subscribeForChangesImpl(AccessEntityType type, co
};
}
bool DiskAccessStorage::hasSubscriptionImpl(const UUID & id) const
bool DiskAccessStorage::hasSubscription(const UUID & id) const
{
std::lock_guard lock{mutex};
auto it = entries_by_id.find(id);
@ -687,7 +712,7 @@ bool DiskAccessStorage::hasSubscriptionImpl(const UUID & id) const
return false;
}
bool DiskAccessStorage::hasSubscriptionImpl(AccessEntityType type) const
bool DiskAccessStorage::hasSubscription(AccessEntityType type) const
{
std::lock_guard lock{mutex};
const auto & handlers = handlers_by_type[static_cast<size_t>(type)];

26
src/Access/DiskAccessStorage.h
View File

@ -24,22 +24,22 @@ public:
bool isPathEqual(const String & directory_path_) const;
void setReadOnly(bool readonly_) { readonly = readonly_; }
bool isReadOnly() const { return readonly; }
bool isReadOnly() const override { return readonly; }
bool exists(const UUID & id) const override;
bool hasSubscription(const UUID & id) const override;
bool hasSubscription(AccessEntityType type) const override;
private:
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
bool existsImpl(const UUID & id) const override;
AccessEntityPtr readImpl(const UUID & id) const override;
String readNameImpl(const UUID & id) const override;
bool canInsertImpl(const AccessEntityPtr & entity) const override;
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(AccessEntityType type) const override;
void clear();
bool readLists();
@ -50,9 +50,9 @@ private:
void listsWritingThreadFunc();
void stopListsWritingThread();
void insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications);
void removeNoLock(const UUID & id, Notifications & notifications);
void updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications);
bool insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, Notifications & notifications);
bool removeNoLock(const UUID & id, bool throw_if_not_exists, Notifications & notifications);
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, Notifications & notifications);
AccessEntityPtr readAccessEntityFromDisk(const UUID & id) const;
void writeAccessEntityToDisk(const UUID & id, const IAccessEntity & entity) const;

537
src/Access/IAccessStorage.cpp
View File

@ -21,8 +21,8 @@ namespace ErrorCodes
extern const int ACCESS_STORAGE_READONLY;
extern const int WRONG_PASSWORD;
extern const int IP_ADDRESS_NOT_ALLOWED;
extern const int AUTHENTICATION_FAILED;
extern const int LOGICAL_ERROR;
extern const int NOT_IMPLEMENTED;
}
@ -32,101 +32,6 @@ namespace
{
return "ID(" + toString(id) + ")";
}
String formatTypeWithNameOrID(const IAccessStorage & storage, const UUID & id)
{
auto entity = storage.tryRead(id);
if (entity)
return entity->formatTypeWithName();
return outputID(id);
}
template <typename Func>
bool tryCall(const Func & function)
{
try
{
function();
return true;
}
catch (...)
{
return false;
}
}
class ErrorsTracker
{
public:
explicit ErrorsTracker(size_t count_) { succeed.reserve(count_); }
template <typename Func>
bool tryCall(const Func & func)
{
try
{
func();
}
catch (Exception & e)
{
if (!exception)
exception.emplace(e);
succeed.push_back(false);
return false;
}
catch (Poco::Exception & e)
{
if (!exception)
exception.emplace(Exception::CreateFromPocoTag{}, e);
succeed.push_back(false);
return false;
}
catch (std::exception & e)
{
if (!exception)
exception.emplace(Exception::CreateFromSTDTag{}, e);
succeed.push_back(false);
return false;
}
succeed.push_back(true);
return true;
}
bool errors() const { return exception.has_value(); }
void showErrors(const char * format, Fn<String(size_t)> auto && get_name_function)
{
if (!exception)
return;
Strings succeeded_names_list;
Strings failed_names_list;
for (size_t i = 0; i != succeed.size(); ++i)
{
String name = get_name_function(i);
if (succeed[i])
succeeded_names_list.emplace_back(name);
else
failed_names_list.emplace_back(name);
}
String succeeded_names = boost::algorithm::join(succeeded_names_list, ", ");
String failed_names = boost::algorithm::join(failed_names_list, ", ");
if (succeeded_names.empty())
succeeded_names = "none";
String error_message = format;
boost::replace_all(error_message, "{succeeded_names}", succeeded_names);
boost::replace_all(error_message, "{failed_names}", failed_names);
exception->addMessage(error_message);
exception->rethrow();
}
private:
std::vector<bool> succeed;
std::optional<Exception> exception;
};
}
@ -175,228 +80,332 @@ std::vector<UUID> IAccessStorage::getIDs(AccessEntityType type, const Strings &
}
bool IAccessStorage::exists(const UUID & id) const
{
return existsImpl(id);
}
AccessEntityPtr IAccessStorage::tryReadBase(const UUID & id) const
{
AccessEntityPtr entity;
auto func = [&] { entity = readImpl(id); };
if (!tryCall(func))
return nullptr;
return entity;
}
String IAccessStorage::readName(const UUID & id) const
{
return readNameImpl(id);
return *readNameImpl(id, /* throw_if_not_exists = */ true);
}
Strings IAccessStorage::readNames(const std::vector<UUID> & ids) const
std::optional<String> IAccessStorage::readName(const UUID & id, bool throw_if_not_exists) const
{
Strings res;
res.reserve(ids.size());
for (const auto & id : ids)
res.emplace_back(readName(id));
return res;
return readNameImpl(id, throw_if_not_exists);
}
std::optional<String> IAccessStorage::tryReadName(const UUID & id) const
{
String name;
auto func = [&] { name = readNameImpl(id); };
if (!tryCall(func))
return {};
return name;
}
Strings IAccessStorage::tryReadNames(const std::vector<UUID> & ids) const
Strings IAccessStorage::readNames(const std::vector<UUID> & ids, bool throw_if_not_exists) const
{
Strings res;
res.reserve(ids.size());
for (const auto & id : ids)
{
if (auto name = tryReadName(id))
if (auto name = readNameImpl(id, throw_if_not_exists))
res.emplace_back(std::move(name).value());
}
return res;
}
UUID IAccessStorage::insert(const AccessEntityPtr & entity)
std::optional<String> IAccessStorage::tryReadName(const UUID & id) const
{
return insertImpl(entity, false);
return readName(id, /* throw_if_not_exists = */ false);
}
std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & multiple_entities)
Strings IAccessStorage::tryReadNames(const std::vector<UUID> & ids) const
{
ErrorsTracker tracker(multiple_entities.size());
return readNames(ids, /* throw_if_not_exists = */ false);
}
std::vector<UUID> ids;
for (const auto & entity : multiple_entities)
std::optional<String> IAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
{
if (auto entity = read(id, throw_if_not_exists))
return entity->getName();
return std::nullopt;
}
UUID IAccessStorage::insert(const AccessEntityPtr & entity)
{
return *insert(entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true);
}
std::optional<UUID> IAccessStorage::insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
{
return insertImpl(entity, replace_if_exists, throw_if_exists);
}
std::vector<UUID> IAccessStorage::insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists, bool throw_if_exists)
{
if (multiple_entities.empty())
return {};
if (multiple_entities.size() == 1)
{
UUID id;
auto func = [&] { id = insertImpl(entity, /* replace_if_exists = */ false); };
if (tracker.tryCall(func))
ids.push_back(id);
if (auto id = insert(multiple_entities[0], replace_if_exists, throw_if_exists))
return {*id};
return {};
}
if (tracker.errors())
std::vector<AccessEntityPtr> successfully_inserted;
try
{
auto get_name_function = [&](size_t i) { return multiple_entities[i]->formatTypeWithName(); };
tracker.showErrors("Couldn't insert {failed_names}. Successfully inserted: {succeeded_names}", get_name_function);
std::vector<UUID> ids;
for (const auto & entity : multiple_entities)
{
if (auto id = insertImpl(entity, replace_if_exists, throw_if_exists))
{
successfully_inserted.push_back(entity);
ids.push_back(*id);
}
}
return ids;
}
catch (Exception & e)
{
/// Try to add more information to the error message.
if (!successfully_inserted.empty())
{
String successfully_inserted_str;
for (const auto & entity : successfully_inserted)
{
if (!successfully_inserted_str.empty())
successfully_inserted_str += ", ";
successfully_inserted_str += entity->formatTypeWithName();
}
e.addMessage("After successfully inserting {}/{}: {}", successfully_inserted.size(), multiple_entities.size(), successfully_inserted_str);
}
e.rethrow();
__builtin_unreachable();
}
return ids;
}
std::optional<UUID> IAccessStorage::tryInsert(const AccessEntityPtr & entity)
{
UUID id;
auto func = [&] { id = insertImpl(entity, /* replace_if_exists = */ false); };
if (!tryCall(func))
return {};
return id;
return insert(entity, /* replace_if_exists = */ false, /* throw_if_exists = */ false);
}
std::vector<UUID> IAccessStorage::tryInsert(const std::vector<AccessEntityPtr> & multiple_entities)
{
std::vector<UUID> ids;
for (const auto & entity : multiple_entities)
{
UUID id;
auto func = [&] { id = insertImpl(entity, /* replace_if_exists = */ false); };
if (tryCall(func))
ids.push_back(id);
}
return ids;
return insert(multiple_entities, /* replace_if_exists = */ false, /* throw_if_exists = */ false);
}
UUID IAccessStorage::insertOrReplace(const AccessEntityPtr & entity)
{
return insertImpl(entity, /* replace_if_exists = */ true);
return *insert(entity, /* replace_if_exists = */ true, /* throw_if_exists = */ false);
}
std::vector<UUID> IAccessStorage::insertOrReplace(const std::vector<AccessEntityPtr> & multiple_entities)
{
ErrorsTracker tracker(multiple_entities.size());
std::vector<UUID> ids;
for (const auto & entity : multiple_entities)
{
UUID id;
auto func = [&] { id = insertImpl(entity, /* replace_if_exists = */ true); };
if (tracker.tryCall(func))
ids.push_back(id);
}
if (tracker.errors())
{
auto get_name_function = [&](size_t i) { return multiple_entities[i]->formatTypeWithName(); };
tracker.showErrors("Couldn't insert {failed_names}. Successfully inserted: {succeeded_names}", get_name_function);
}
return ids;
return insert(multiple_entities, /* replace_if_exists = */ true, /* throw_if_exists = */ false);
}
void IAccessStorage::remove(const UUID & id)
std::optional<UUID> IAccessStorage::insertImpl(const AccessEntityPtr & entity, bool, bool)
{
removeImpl(id);
if (isReadOnly())
throwReadonlyCannotInsert(entity->getType(), entity->getName());
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "insertImpl() is not implemented in {}", getStorageType());
}
void IAccessStorage::remove(const std::vector<UUID> & ids)
bool IAccessStorage::remove(const UUID & id, bool throw_if_not_exists)
{
ErrorsTracker tracker(ids.size());
return removeImpl(id, throw_if_not_exists);
}
for (const auto & id : ids)
std::vector<UUID> IAccessStorage::remove(const std::vector<UUID> & ids, bool throw_if_not_exists)
{
if (ids.empty())
return {};
if (ids.size() == 1)
return remove(ids[0], throw_if_not_exists) ? ids : std::vector<UUID>{};
Strings removed_names;
try
{
auto func = [&] { removeImpl(id); };
tracker.tryCall(func);
std::vector<UUID> removed_ids;
std::vector<UUID> readonly_ids;
/// First we call remove() for non-readonly entities.
for (const auto & id : ids)
{
if (isReadOnly(id))
readonly_ids.push_back(id);
else
{
auto name = tryReadName(id);
if (remove(id, throw_if_not_exists))
{
removed_ids.push_back(id);
if (name)
removed_names.push_back(std::move(name).value());
}
}
}
/// For readonly entities we're still going to call remove() because
/// isReadOnly(id) could change and even if it's not then a storage-specific
/// implementation of removeImpl() will probably generate a better error message.
for (const auto & id : readonly_ids)
{
auto name = tryReadName(id);
if (remove(id, throw_if_not_exists))
{
removed_ids.push_back(id);
if (name)
removed_names.push_back(std::move(name).value());
}
}
return removed_ids;
}
if (tracker.errors())
catch (Exception & e)
{
auto get_name_function = [&](size_t i) { return formatTypeWithNameOrID(*this, ids[i]); };
tracker.showErrors("Couldn't remove {failed_names}. Successfully removed: {succeeded_names}", get_name_function);
/// Try to add more information to the error message.
if (!removed_names.empty())
{
String removed_names_str;
for (const auto & name : removed_names)
{
if (!removed_names_str.empty())
removed_names_str += ", ";
removed_names_str += backQuote(name);
}
e.addMessage("After successfully removing {}/{}: {}", removed_names.size(), ids.size(), removed_names_str);
}
e.rethrow();
__builtin_unreachable();
}
}
bool IAccessStorage::tryRemove(const UUID & id)
{
auto func = [&] { removeImpl(id); };
return tryCall(func);
return remove(id, /* throw_if_not_exists = */ false);
}
std::vector<UUID> IAccessStorage::tryRemove(const std::vector<UUID> & ids)
{
std::vector<UUID> removed_ids;
for (const auto & id : ids)
{
auto func = [&] { removeImpl(id); };
if (tryCall(func))
removed_ids.push_back(id);
}
return removed_ids;
return remove(ids, /* throw_if_not_exists = */ false);
}
void IAccessStorage::update(const UUID & id, const UpdateFunc & update_func)
bool IAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists)
{
updateImpl(id, update_func);
if (isReadOnly(id))
{
auto entity = read(id, throw_if_not_exists);
if (!entity)
return false;
throwReadonlyCannotRemove(entity->getType(), entity->getName());
}
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "removeImpl() is not implemented in {}", getStorageType());
}
void IAccessStorage::update(const std::vector<UUID> & ids, const UpdateFunc & update_func)
bool IAccessStorage::update(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists)
{
ErrorsTracker tracker(ids.size());
return updateImpl(id, update_func, throw_if_not_exists);
}
for (const auto & id : ids)
std::vector<UUID> IAccessStorage::update(const std::vector<UUID> & ids, const UpdateFunc & update_func, bool throw_if_not_exists)
{
if (ids.empty())
return {};
if (ids.size() == 1)
return update(ids[0], update_func, throw_if_not_exists) ? ids : std::vector<UUID>{};
Strings names_of_updated;
try
{
auto func = [&] { updateImpl(id, update_func); };
tracker.tryCall(func);
std::vector<UUID> ids_of_updated;
std::vector<UUID> readonly_ids;
/// First we call update() for non-readonly entities.
for (const auto & id : ids)
{
if (isReadOnly(id))
readonly_ids.push_back(id);
else
{
auto name = tryReadName(id);
if (update(id, update_func, throw_if_not_exists))
{
ids_of_updated.push_back(id);
if (name)
names_of_updated.push_back(std::move(name).value());
}
}
}
/// For readonly entities we're still going to call update() because
/// isReadOnly(id) could change and even if it's not then a storage-specific
/// implementation of updateImpl() will probably generate a better error message.
for (const auto & id : readonly_ids)
{
auto name = tryReadName(id);
if (update(id, update_func, throw_if_not_exists))
{
ids_of_updated.push_back(id);
if (name)
names_of_updated.push_back(std::move(name).value());
}
}
return ids_of_updated;
}
if (tracker.errors())
catch (Exception & e)
{
auto get_name_function = [&](size_t i) { return formatTypeWithNameOrID(*this, ids[i]); };
tracker.showErrors("Couldn't update {failed_names}. Successfully updated: {succeeded_names}", get_name_function);
/// Try to add more information to the error message.
if (!names_of_updated.empty())
{
String names_of_updated_str;
for (const auto & name : names_of_updated)
{
if (!names_of_updated_str.empty())
names_of_updated_str += ", ";
names_of_updated_str += backQuote(name);
}
e.addMessage("After successfully updating {}/{}: {}", names_of_updated.size(), ids.size(), names_of_updated_str);
}
e.rethrow();
__builtin_unreachable();
}
}
bool IAccessStorage::tryUpdate(const UUID & id, const UpdateFunc & update_func)
{
auto func = [&] { updateImpl(id, update_func); };
return tryCall(func);
return update(id, update_func, /* throw_if_not_exists = */ false);
}
std::vector<UUID> IAccessStorage::tryUpdate(const std::vector<UUID> & ids, const UpdateFunc & update_func)
{
std::vector<UUID> updated_ids;
for (const auto & id : ids)
return update(ids, update_func, /* throw_if_not_exists = */ false);
}
bool IAccessStorage::updateImpl(const UUID & id, const UpdateFunc &, bool throw_if_not_exists)
{
if (isReadOnly(id))
{
auto func = [&] { updateImpl(id, update_func); };
if (tryCall(func))
updated_ids.push_back(id);
auto entity = read(id, throw_if_not_exists);
if (!entity)
return false;
throwReadonlyCannotUpdate(entity->getType(), entity->getName());
}
return updated_ids;
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "updateImpl() is not implemented in {}", getStorageType());
}
@ -421,18 +430,6 @@ scope_guard IAccessStorage::subscribeForChanges(const std::vector<UUID> & ids, c
}
bool IAccessStorage::hasSubscription(AccessEntityType type) const
{
return hasSubscriptionImpl(type);
}
bool IAccessStorage::hasSubscription(const UUID & id) const
{
return hasSubscriptionImpl(id);
}
void IAccessStorage::notify(const Notifications & notifications)
{
for (const auto & [fn, id, new_entity] : notifications)
@ -440,50 +437,53 @@ void IAccessStorage::notify(const Notifications & notifications)
}
UUID IAccessStorage::login(
const Credentials & credentials,
const Poco::Net::IPAddress & address,
const ExternalAuthenticators & external_authenticators,
bool replace_exception_with_cannot_authenticate) const
{
try
{
return loginImpl(credentials, address, external_authenticators);
}
catch (...)
{
if (!replace_exception_with_cannot_authenticate)
throw;
tryLogCurrentException(getLogger(), "from: " + address.toString() + ", user: " + credentials.getUserName() + ": Authentication failed");
throwCannotAuthenticate(credentials.getUserName());
}
}
UUID IAccessStorage::loginImpl(
UUID IAccessStorage::authenticate(
const Credentials & credentials,
const Poco::Net::IPAddress & address,
const ExternalAuthenticators & external_authenticators) const
{
return *authenticateImpl(credentials, address, external_authenticators, /* throw_if_user_not_exists = */ true);
}
std::optional<UUID> IAccessStorage::authenticate(
const Credentials & credentials,
const Poco::Net::IPAddress & address,
const ExternalAuthenticators & external_authenticators,
bool throw_if_user_not_exists) const
{
return authenticateImpl(credentials, address, external_authenticators, throw_if_user_not_exists);
}
std::optional<UUID> IAccessStorage::authenticateImpl(
const Credentials & credentials,
const Poco::Net::IPAddress & address,
const ExternalAuthenticators & external_authenticators,
bool throw_if_user_not_exists) const
{
if (auto id = find<User>(credentials.getUserName()))
{
if (auto user = tryRead<User>(*id))
{
if (!isAddressAllowedImpl(*user, address))
if (!isAddressAllowed(*user, address))
throwAddressNotAllowed(address);
if (!areCredentialsValidImpl(*user, credentials, external_authenticators))
if (!areCredentialsValid(*user, credentials, external_authenticators))
throwInvalidCredentials();
return *id;
return id;
}
}
throwNotFound(AccessEntityType::USER, credentials.getUserName());
if (throw_if_user_not_exists)
throwNotFound(AccessEntityType::USER, credentials.getUserName());
else
return std::nullopt;
}
bool IAccessStorage::areCredentialsValidImpl(
bool IAccessStorage::areCredentialsValid(
const User & user,
const Credentials & credentials,
const ExternalAuthenticators & external_authenticators) const
@ -498,24 +498,12 @@ bool IAccessStorage::areCredentialsValidImpl(
}
bool IAccessStorage::isAddressAllowedImpl(const User & user, const Poco::Net::IPAddress & address) const
bool IAccessStorage::isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const
{
return user.allowed_client_hosts.contains(address);
}
UUID IAccessStorage::getIDOfLoggedUser(const String & user_name) const
{
return getIDOfLoggedUserImpl(user_name);
}
UUID IAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const
{
return getID<User>(user_name);
}
UUID IAccessStorage::generateRandomID()
{
static Poco::UUIDGenerator generator;
@ -615,11 +603,4 @@ void IAccessStorage::throwInvalidCredentials()
throw Exception("Invalid credentials", ErrorCodes::WRONG_PASSWORD);
}
void IAccessStorage::throwCannotAuthenticate(const String & user_name)
{
/// We use the same message for all authentication failures because we don't want to give away any unnecessary information for security reasons,
/// only the log will show the exact reason.
throw Exception(user_name + ": Authentication failed: password is incorrect or there is no user with such name", ErrorCodes::AUTHENTICATION_FAILED);
}
}

91
src/Access/IAccessStorage.h
View File

@ -34,6 +34,12 @@ public:
/// Returns a JSON with the parameters of the storage. It's up to the storage type to fill the JSON.
virtual String getStorageParamsJSON() const { return "{}"; }
/// Returns true if this storage is readonly.
virtual bool isReadOnly() const { return false; }
/// Returns true if this entity is readonly.
virtual bool isReadOnly(const UUID &) const { return isReadOnly(); }
/// Returns the identifiers of all the entities of a specified type contained in the storage.
std::vector<UUID> findAll(AccessEntityType type) const;
@ -63,14 +69,14 @@ public:
std::vector<UUID> getIDs(const Strings & names) const { return getIDs(EntityClassT::TYPE, names); }
/// Returns whether there is an entity with such identifier in the storage.
bool exists(const UUID & id) const;
virtual bool exists(const UUID & id) const = 0;
/// Reads an entity. Throws an exception if not found.
template <typename EntityClassT = IAccessEntity>
std::shared_ptr<const EntityClassT> read(const UUID & id) const;
std::shared_ptr<const EntityClassT> read(const UUID & id, bool throw_if_not_exists = true) const;
template <typename EntityClassT = IAccessEntity>
std::shared_ptr<const EntityClassT> read(const String & name) const;
std::shared_ptr<const EntityClassT> read(const String & name, bool throw_if_not_exists = true) const;
/// Reads an entity. Returns nullptr if not found.
template <typename EntityClassT = IAccessEntity>
@ -81,18 +87,16 @@ public:
/// Reads only name of an entity.
String readName(const UUID & id) const;
Strings readNames(const std::vector<UUID> & ids) const;
std::optional<String> readName(const UUID & id, bool throw_if_not_exists) const;
Strings readNames(const std::vector<UUID> & ids, bool throw_if_not_exists = true) const;
std::optional<String> tryReadName(const UUID & id) const;
Strings tryReadNames(const std::vector<UUID> & ids) const;
/// Returns true if a specified entity can be inserted into this storage.
/// This function doesn't check whether there are no entities with such name in the storage.
bool canInsert(const AccessEntityPtr & entity) const { return canInsertImpl(entity); }
/// Inserts an entity to the storage. Returns ID of a new entry in the storage.
/// Throws an exception if the specified name already exists.
UUID insert(const AccessEntityPtr & entity);
std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities);
std::optional<UUID> insert(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
std::vector<UUID> insert(const std::vector<AccessEntityPtr> & multiple_entities, bool replace_if_exists = false, bool throw_if_exists = true);
/// Inserts an entity to the storage. Returns ID of a new entry in the storage.
std::optional<UUID> tryInsert(const AccessEntityPtr & entity);
@ -104,8 +108,8 @@ public:
std::vector<UUID> insertOrReplace(const std::vector<AccessEntityPtr> & multiple_entities);
/// Removes an entity from the storage. Throws an exception if couldn't remove.
void remove(const UUID & id);
void remove(const std::vector<UUID> & ids);
bool remove(const UUID & id, bool throw_if_not_exists = true);
std::vector<UUID> remove(const std::vector<UUID> & ids, bool throw_if_not_exists = true);
/// Removes an entity from the storage. Returns false if couldn't remove.
bool tryRemove(const UUID & id);
@ -116,8 +120,8 @@ public:
using UpdateFunc = std::function<AccessEntityPtr(const AccessEntityPtr &)>;
/// Updates an entity stored in the storage. Throws an exception if couldn't update.
void update(const UUID & id, const UpdateFunc & update_func);
void update(const std::vector<UUID> & ids, const UpdateFunc & update_func);
bool update(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists = true);
std::vector<UUID> update(const std::vector<UUID> & ids, const UpdateFunc & update_func, bool throw_if_not_exists = true);
/// Updates an entity stored in the storage. Returns false if couldn't update.
bool tryUpdate(const UUID & id, const UpdateFunc & update_func);
@ -139,35 +143,27 @@ public:
scope_guard subscribeForChanges(const UUID & id, const OnChangedHandler & handler) const;
scope_guard subscribeForChanges(const std::vector<UUID> & ids, const OnChangedHandler & handler) const;
bool hasSubscription(AccessEntityType type) const;
bool hasSubscription(const UUID & id) const;
virtual bool hasSubscription(AccessEntityType type) const = 0;
virtual bool hasSubscription(const UUID & id) const = 0;
/// Finds a user, check the provided credentials and returns the ID of the user if they are valid.
/// Throws an exception if no such user or credentials are invalid.
UUID login(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool replace_exception_with_cannot_authenticate = true) const;
/// Returns the ID of a user who has logged in (maybe on another node).
/// The function assumes that the password has been already checked somehow, so we can skip checking it now.
UUID getIDOfLoggedUser(const String & user_name) const;
UUID authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const;
std::optional<UUID> authenticate(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists) const;
protected:
virtual std::optional<UUID> findImpl(AccessEntityType type, const String & name) const = 0;
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const = 0;
virtual bool existsImpl(const UUID & id) const = 0;
virtual AccessEntityPtr readImpl(const UUID & id) const = 0;
virtual String readNameImpl(const UUID & id) const = 0;
virtual bool canInsertImpl(const AccessEntityPtr & entity) const = 0;
virtual UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) = 0;
virtual void removeImpl(const UUID & id) = 0;
virtual void updateImpl(const UUID & id, const UpdateFunc & update_func) = 0;
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const = 0;
virtual std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const;
virtual std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
virtual bool removeImpl(const UUID & id, bool throw_if_not_exists);
virtual bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
virtual scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const = 0;
virtual scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const = 0;
virtual bool hasSubscriptionImpl(const UUID & id) const = 0;
virtual bool hasSubscriptionImpl(AccessEntityType type) const = 0;
virtual UUID loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const;
virtual bool areCredentialsValidImpl(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const;
virtual bool isAddressAllowedImpl(const User & user, const Poco::Net::IPAddress & address) const;
virtual UUID getIDOfLoggedUserImpl(const String & user_name) const;
virtual std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists) const;
virtual bool areCredentialsValid(const User & user, const Credentials & credentials, const ExternalAuthenticators & external_authenticators) const;
virtual bool isAddressAllowed(const User & user, const Poco::Net::IPAddress & address) const;
static UUID generateRandomID();
Poco::Logger * getLogger() const;
@ -184,30 +180,28 @@ protected:
[[noreturn]] void throwReadonlyCannotRemove(AccessEntityType type, const String & name) const;
[[noreturn]] static void throwAddressNotAllowed(const Poco::Net::IPAddress & address);
[[noreturn]] static void throwInvalidCredentials();
[[noreturn]] static void throwCannotAuthenticate(const String & user_name);
using Notification = std::tuple<OnChangedHandler, UUID, AccessEntityPtr>;
using Notifications = std::vector<Notification>;
static void notify(const Notifications & notifications);
private:
AccessEntityPtr tryReadBase(const UUID & id) const;
const String storage_name;
mutable std::atomic<Poco::Logger *> log = nullptr;
};
template <typename EntityClassT>
std::shared_ptr<const EntityClassT> IAccessStorage::read(const UUID & id) const
std::shared_ptr<const EntityClassT> IAccessStorage::read(const UUID & id, bool throw_if_not_exists) const
{
auto entity = readImpl(id);
auto entity = readImpl(id, throw_if_not_exists);
if constexpr (std::is_same_v<EntityClassT, IAccessEntity>)
return entity;
else
{
auto ptr = typeid_cast<std::shared_ptr<const EntityClassT>>(entity);
if (ptr)
if (!entity)
return nullptr;
if (auto ptr = typeid_cast<std::shared_ptr<const EntityClassT>>(entity))
return ptr;
throwBadCast(id, entity->getType(), entity->getName(), EntityClassT::TYPE);
}
@ -215,26 +209,27 @@ std::shared_ptr<const EntityClassT> IAccessStorage::read(const UUID & id) const
template <typename EntityClassT>
std::shared_ptr<const EntityClassT> IAccessStorage::read(const String & name) const
std::shared_ptr<const EntityClassT> IAccessStorage::read(const String & name, bool throw_if_not_exists) const
{
return read<EntityClassT>(getID<EntityClassT>(name));
if (auto id = find<EntityClassT>(name))
return read<EntityClassT>(*id, throw_if_not_exists);
if (throw_if_not_exists)
throwNotFound(EntityClassT::TYPE, name);
else
return nullptr;
}
template <typename EntityClassT>
std::shared_ptr<const EntityClassT> IAccessStorage::tryRead(const UUID & id) const
{
auto entity = tryReadBase(id);
if (!entity)
return nullptr;
return typeid_cast<std::shared_ptr<const EntityClassT>>(entity);
return read<EntityClassT>(id, false);
}
template <typename EntityClassT>
std::shared_ptr<const EntityClassT> IAccessStorage::tryRead(const String & name) const
{
auto id = find<EntityClassT>(name);
return id ? tryRead<EntityClassT>(*id) : nullptr;
return read<EntityClassT>(name, false);
}
}

91
src/Access/LDAPAccessStorage.cpp
View File

@ -426,52 +426,24 @@ std::vector<UUID> LDAPAccessStorage::findAllImpl(AccessEntityType type) const
}
bool LDAPAccessStorage::existsImpl(const UUID & id) const
bool LDAPAccessStorage::exists(const UUID & id) const
{
std::scoped_lock lock(mutex);
return memory_storage.exists(id);
}
AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id) const
AccessEntityPtr LDAPAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
{
std::scoped_lock lock(mutex);
return memory_storage.read(id);
return memory_storage.read(id, throw_if_not_exists);
}
String LDAPAccessStorage::readNameImpl(const UUID & id) const
std::optional<String> LDAPAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
{
std::scoped_lock lock(mutex);
return memory_storage.readName(id);
}
bool LDAPAccessStorage::canInsertImpl(const AccessEntityPtr &) const
{
return false;
}
UUID LDAPAccessStorage::insertImpl(const AccessEntityPtr & entity, bool)
{
throwReadonlyCannotInsert(entity->getType(), entity->getName());
}
void LDAPAccessStorage::removeImpl(const UUID & id)
{
std::scoped_lock lock(mutex);
auto entity = read(id);
throwReadonlyCannotRemove(entity->getType(), entity->getName());
}
void LDAPAccessStorage::updateImpl(const UUID & id, const UpdateFunc &)
{
std::scoped_lock lock(mutex);
auto entity = read(id);
throwReadonlyCannotUpdate(entity->getType(), entity->getName());
return memory_storage.readName(id, throw_if_not_exists);
}
@ -489,20 +461,24 @@ scope_guard LDAPAccessStorage::subscribeForChangesImpl(AccessEntityType type, co
}
bool LDAPAccessStorage::hasSubscriptionImpl(const UUID & id) const
bool LDAPAccessStorage::hasSubscription(const UUID & id) const
{
std::scoped_lock lock(mutex);
return memory_storage.hasSubscription(id);
}
bool LDAPAccessStorage::hasSubscriptionImpl(AccessEntityType type) const
bool LDAPAccessStorage::hasSubscription(AccessEntityType type) const
{
std::scoped_lock lock(mutex);
return memory_storage.hasSubscription(type);
}
UUID LDAPAccessStorage::loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const
std::optional<UUID> LDAPAccessStorage::authenticateImpl(
const Credentials & credentials,
const Poco::Net::IPAddress & address,
const ExternalAuthenticators & external_authenticators,
bool /* throw_if_user_not_exists */) const
{
std::scoped_lock lock(mutex);
LDAPClient::SearchResultsList external_roles;
@ -511,16 +487,19 @@ UUID LDAPAccessStorage::loginImpl(const Credentials & credentials, const Poco::N
{
auto user = memory_storage.read<User>(*id);
if (!isAddressAllowedImpl(*user, address))
if (!isAddressAllowed(*user, address))
throwAddressNotAllowed(address);
if (typeid_cast<const AlwaysAllowCredentials *>(&credentials))
return id;
if (!areLDAPCredentialsValidNoLock(*user, credentials, external_authenticators, external_roles))
throwInvalidCredentials();
// Just in case external_roles are changed. This will be no-op if they are not.
updateAssignedRolesNoLock(*id, user->getName(), external_roles);
return *id;
return id;
}
else
{
@ -530,9 +509,16 @@ UUID LDAPAccessStorage::loginImpl(const Credentials & credentials, const Poco::N
user->auth_data = AuthenticationData(AuthenticationType::LDAP);
user->auth_data.setLDAPServerName(ldap_server_name);
if (!isAddressAllowedImpl(*user, address))
if (!isAddressAllowed(*user, address))
throwAddressNotAllowed(address);
if (typeid_cast<const AlwaysAllowCredentials *>(&credentials))
{
// TODO: mapped external roles are not available here. Without a password we can't authenticate and retrieve roles from LDAP server.
assignRolesNoLock(*user, external_roles);
return memory_storage.insert(user);
}
if (!areLDAPCredentialsValidNoLock(*user, credentials, external_authenticators, external_roles))
throwInvalidCredentials();
@ -541,31 +527,4 @@ UUID LDAPAccessStorage::loginImpl(const Credentials & credentials, const Poco::N
return memory_storage.insert(user);
}
}
UUID LDAPAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const
{
std::scoped_lock lock(mutex);
auto id = memory_storage.find<User>(user_name);
if (id)
{
return *id;
}
else
{
// User does not exist, so we create one, and add it pretending that the authentication is successful.
auto user = std::make_shared<User>();
user->setName(user_name);
user->auth_data = AuthenticationData(AuthenticationType::LDAP);
user->auth_data.setLDAPServerName(ldap_server_name);
LDAPClient::SearchResultsList external_roles;
// TODO: mapped external roles are not available here. Without a password we can't authenticate and retrieve roles from LDAP server.
assignRolesNoLock(*user, external_roles);
return memory_storage.insert(user);
}
}
}

18
src/Access/LDAPAccessStorage.h
View File

@ -40,23 +40,19 @@ public:
public: // IAccessStorage implementations.
virtual const char * getStorageType() const override;
virtual String getStorageParamsJSON() const override;
virtual bool isReadOnly() const override { return true; }
virtual bool exists(const UUID & id) const override;
virtual bool hasSubscription(const UUID & id) const override;
virtual bool hasSubscription(AccessEntityType type) const override;
private: // IAccessStorage implementations.
virtual std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
virtual std::vector<UUID> findAllImpl(AccessEntityType type) const override;
virtual bool existsImpl(const UUID & id) const override;
virtual AccessEntityPtr readImpl(const UUID & id) const override;
virtual String readNameImpl(const UUID & id) const override;
virtual bool canInsertImpl(const AccessEntityPtr &) const override;
virtual UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
virtual void removeImpl(const UUID & id) override;
virtual void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
virtual AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
virtual std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const override;
virtual scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
virtual scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override;
virtual bool hasSubscriptionImpl(const UUID & id) const override;
virtual bool hasSubscriptionImpl(AccessEntityType type) const override;
virtual UUID loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override;
virtual UUID getIDOfLoggedUserImpl(const String & user_name) const override;
virtual std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists) const override;
private:
void setConfiguration(AccessControl * access_control_, const Poco::Util::AbstractConfiguration & config, const String & prefix);

102
src/Access/MemoryAccessStorage.cpp
View File

@ -38,64 +38,72 @@ std::vector<UUID> MemoryAccessStorage::findAllImpl(AccessEntityType type) const
}
bool MemoryAccessStorage::existsImpl(const UUID & id) const
bool MemoryAccessStorage::exists(const UUID & id) const
{
std::lock_guard lock{mutex};
return entries_by_id.count(id);
}
AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id) const
AccessEntityPtr MemoryAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
{
std::lock_guard lock{mutex};
auto it = entries_by_id.find(id);
if (it == entries_by_id.end())
throwNotFound(id);
{
if (throw_if_not_exists)
throwNotFound(id);
else
return nullptr;
}
const Entry & entry = it->second;
return entry.entity;
}
String MemoryAccessStorage::readNameImpl(const UUID & id) const
{
return readImpl(id)->getName();
}
UUID MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists)
std::optional<UUID> MemoryAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
UUID id = generateRandomID();
std::lock_guard lock{mutex};
insertNoLock(id, new_entity, replace_if_exists, notifications);
return id;
if (insertNoLock(id, new_entity, replace_if_exists, throw_if_exists, notifications))
return id;
return std::nullopt;
}
void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, Notifications & notifications)
bool MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists, Notifications & notifications)
{
const String & name = new_entity->getName();
AccessEntityType type = new_entity->getType();
/// Check that we can insert.
auto it = entries_by_id.find(id);
if (it != entries_by_id.end())
auto & entries_by_name = entries_by_name_and_type[static_cast<size_t>(type)];
auto it_by_name = entries_by_name.find(name);
bool name_collision = (it_by_name != entries_by_name.end());
if (name_collision && !replace_if_exists)
{
const auto & existing_entry = it->second;
if (throw_if_exists)
throwNameCollisionCannotInsert(type, name);
else
return false;
}
auto it_by_id = entries_by_id.find(id);
if (it_by_id != entries_by_id.end())
{
const auto & existing_entry = it_by_id->second;
throwIDCollisionCannotInsert(id, type, name, existing_entry.entity->getType(), existing_entry.entity->getName());
}
auto & entries_by_name = entries_by_name_and_type[static_cast<size_t>(type)];
auto it2 = entries_by_name.find(name);
if (it2 != entries_by_name.end())
if (name_collision && replace_if_exists)
{
const auto & existing_entry = *(it2->second);
if (replace_if_exists)
removeNoLock(existing_entry.id, notifications);
else
throwNameCollisionCannotInsert(type, name);
const auto & existing_entry = *(it_by_name->second);
removeNoLock(existing_entry.id, /* throw_if_not_exists = */ false, notifications);
}
/// Do insertion.
@ -104,24 +112,30 @@ void MemoryAccessStorage::insertNoLock(const UUID & id, const AccessEntityPtr &
entry.entity = new_entity;
entries_by_name[name] = &entry;
prepareNotifications(entry, false, notifications);
return true;
}
void MemoryAccessStorage::removeImpl(const UUID & id)
bool MemoryAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists)
{
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
std::lock_guard lock{mutex};
removeNoLock(id, notifications);
return removeNoLock(id, throw_if_not_exists, notifications);
}
void MemoryAccessStorage::removeNoLock(const UUID & id, Notifications & notifications)
bool MemoryAccessStorage::removeNoLock(const UUID & id, bool throw_if_not_exists, Notifications & notifications)
{
auto it = entries_by_id.find(id);
if (it == entries_by_id.end())
throwNotFound(id);
{
if (throw_if_not_exists)
throwNotFound(id);
else
return false;
}
Entry & entry = it->second;
const String & name = entry.entity->getName();
@ -133,24 +147,30 @@ void MemoryAccessStorage::removeNoLock(const UUID & id, Notifications & notifica
auto & entries_by_name = entries_by_name_and_type[static_cast<size_t>(type)];
entries_by_name.erase(name);
entries_by_id.erase(it);
return true;
}
void MemoryAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func)
bool MemoryAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists)
{
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
std::lock_guard lock{mutex};
updateNoLock(id, update_func, notifications);
return updateNoLock(id, update_func, throw_if_not_exists, notifications);
}
void MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications)
bool MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, Notifications & notifications)
{
auto it = entries_by_id.find(id);
if (it == entries_by_id.end())
throwNotFound(id);
{
if (throw_if_not_exists)
throwNotFound(id);
else
return false;
}
Entry & entry = it->second;
auto old_entity = entry.entity;
@ -160,7 +180,7 @@ void MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat
throwBadCast(id, new_entity->getType(), new_entity->getName(), old_entity->getType());
if (*new_entity == *old_entity)
return;
return true;
entry.entity = new_entity;
@ -176,6 +196,7 @@ void MemoryAccessStorage::updateNoLock(const UUID & id, const UpdateFunc & updat
}
prepareNotifications(entry, false, notifications);
return true;
}
@ -235,7 +256,7 @@ void MemoryAccessStorage::setAllNoLock(const std::vector<std::pair<UUID, AccessE
boost::container::flat_set<UUID> ids_to_remove = std::move(not_used_ids);
boost::range::copy(conflicting_ids, std::inserter(ids_to_remove, ids_to_remove.end()));
for (const auto & id : ids_to_remove)
removeNoLock(id, notifications);
removeNoLock(id, /* throw_if_not_exists = */ false, notifications);
/// Insert or update entities.
for (const auto & [id, entity] : all_entities)
@ -246,11 +267,16 @@ void MemoryAccessStorage::setAllNoLock(const std::vector<std::pair<UUID, AccessE
if (*(it->second.entity) != *entity)
{
const AccessEntityPtr & changed_entity = entity;
updateNoLock(id, [&changed_entity](const AccessEntityPtr &) { return changed_entity; }, notifications);
updateNoLock(id,
[&changed_entity](const AccessEntityPtr &) { return changed_entity; },
/* throw_if_not_exists = */ true,
notifications);
}
}
else
insertNoLock(id, entity, false, notifications);
{
insertNoLock(id, entity, /* replace_if_exists = */ false, /* throw_if_exists = */ true, notifications);
}
}
}
@ -304,7 +330,7 @@ scope_guard MemoryAccessStorage::subscribeForChangesImpl(const UUID & id, const
}
bool MemoryAccessStorage::hasSubscriptionImpl(const UUID & id) const
bool MemoryAccessStorage::hasSubscription(const UUID & id) const
{
std::lock_guard lock{mutex};
auto it = entries_by_id.find(id);
@ -317,7 +343,7 @@ bool MemoryAccessStorage::hasSubscriptionImpl(const UUID & id) const
}
bool MemoryAccessStorage::hasSubscriptionImpl(AccessEntityType type) const
bool MemoryAccessStorage::hasSubscription(AccessEntityType type) const
{
std::lock_guard lock{mutex};
const auto & handlers = handlers_by_type[static_cast<size_t>(type)];

23
src/Access/MemoryAccessStorage.h
View File

@ -23,20 +23,19 @@ public:
void setAll(const std::vector<AccessEntityPtr> & all_entities);
void setAll(const std::vector<std::pair<UUID, AccessEntityPtr>> & all_entities);
bool exists(const UUID & id) const override;
bool hasSubscription(const UUID & id) const override;
bool hasSubscription(AccessEntityType type) const override;
private:
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
bool existsImpl(const UUID & id) const override;
AccessEntityPtr readImpl(const UUID & id) const override;
String readNameImpl(const UUID & id) const override;
bool canInsertImpl(const AccessEntityPtr &) const override { return true; }
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(AccessEntityType type) const override;
struct Entry
{
@ -45,9 +44,9 @@ private:
mutable std::list<OnChangedHandler> handlers_by_id;
};
void insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, Notifications & notifications);
void removeNoLock(const UUID & id, Notifications & notifications);
void updateNoLock(const UUID & id, const UpdateFunc & update_func, Notifications & notifications);
bool insertNoLock(const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists, Notifications & notifications);
bool removeNoLock(const UUID & id, bool throw_if_not_exists, Notifications & notifications);
bool updateNoLock(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists, Notifications & notifications);
void setAllNoLock(const std::vector<std::pair<UUID, AccessEntityPtr>> & all_entities, Notifications & notifications);
void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const;

165
src/Access/MultipleAccessStorage.cpp
View File

@ -13,8 +13,8 @@ namespace DB
{
namespace ErrorCodes
{
extern const int ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND;
extern const int ACCESS_ENTITY_ALREADY_EXISTS;
extern const int ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND;
}
using Storage = IAccessStorage;
@ -129,7 +129,7 @@ std::vector<UUID> MultipleAccessStorage::findAllImpl(AccessEntityType type) cons
}
bool MultipleAccessStorage::existsImpl(const UUID & id) const
bool MultipleAccessStorage::exists(const UUID & id) const
{
return findStorage(id) != nullptr;
}
@ -180,39 +180,59 @@ ConstStoragePtr MultipleAccessStorage::getStorage(const UUID & id) const
return const_cast<MultipleAccessStorage *>(this)->getStorage(id);
}
AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id) const
AccessEntityPtr MultipleAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
{
return getStorage(id)->read(id);
if (auto storage = findStorage(id))
return storage->read(id, throw_if_not_exists);
if (throw_if_not_exists)
throwNotFound(id);
else
return nullptr;
}
String MultipleAccessStorage::readNameImpl(const UUID & id) const
std::optional<String> MultipleAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
{
return getStorage(id)->readName(id);
if (auto storage = findStorage(id))
return storage->readName(id, throw_if_not_exists);
if (throw_if_not_exists)
throwNotFound(id);
else
return std::nullopt;
}
bool MultipleAccessStorage::canInsertImpl(const AccessEntityPtr & entity) const
bool MultipleAccessStorage::isReadOnly() const
{
auto storages = getStoragesInternal();
for (const auto & storage : *storages)
{
if (storage->canInsert(entity))
return true;
if (!storage->isReadOnly())
return false;
}
return true;
}
bool MultipleAccessStorage::isReadOnly(const UUID & id) const
{
auto storage = findStorage(id);
if (storage)
return storage->isReadOnly(id);
return false;
}
UUID MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists)
std::optional<UUID> MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists)
{
auto storages = getStoragesInternal();
std::shared_ptr<IAccessStorage> storage_for_insertion;
auto storages = getStoragesInternal();
for (const auto & storage : *storages)
{
if (storage->canInsert(entity) ||
storage->find(entity->getType(), entity->getName()))
if (!storage->isReadOnly() || storage->find(entity->getType(), entity->getName()))
{
storage_for_insertion = storage;
break;
@ -220,49 +240,73 @@ UUID MultipleAccessStorage::insertImpl(const AccessEntityPtr & entity, bool repl
}
if (!storage_for_insertion)
throw Exception("Not found a storage to insert " + entity->formatTypeWithName(), ErrorCodes::ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND);
{
throw Exception(
ErrorCodes::ACCESS_STORAGE_FOR_INSERTION_NOT_FOUND,
"Could not insert {} because there is no writeable access storage in {}",
entity->formatTypeWithName(),
getStorageName());
}
auto id = replace_if_exists ? storage_for_insertion->insertOrReplace(entity) : storage_for_insertion->insert(entity);
std::lock_guard lock{mutex};
ids_cache.set(id, storage_for_insertion);
auto id = storage_for_insertion->insert(entity, replace_if_exists, throw_if_exists);
if (id)
{
std::lock_guard lock{mutex};
ids_cache.set(*id, storage_for_insertion);
}
return id;
}
void MultipleAccessStorage::removeImpl(const UUID & id)
bool MultipleAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists)
{
getStorage(id)->remove(id);
if (auto storage = findStorage(id))
return storage->remove(id, throw_if_not_exists);
if (throw_if_not_exists)
throwNotFound(id);
else
return false;
}
void MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func)
bool MultipleAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists)
{
auto storage_for_updating = getStorage(id);
auto storage_for_updating = findStorage(id);
if (!storage_for_updating)
{
if (throw_if_not_exists)
throwNotFound(id);
else
return false;
}
/// If the updating involves renaming check that the renamed entity will be accessible by name.
auto storages = getStoragesInternal();
if ((storages->size() > 1) && (storages->front() != storage_for_updating))
{
auto old_entity = storage_for_updating->read(id);
auto new_entity = update_func(old_entity);
if (new_entity->getName() != old_entity->getName())
if (auto old_entity = storage_for_updating->tryRead(id))
{
for (const auto & storage : *storages)
auto new_entity = update_func(old_entity);
if (new_entity->getName() != old_entity->getName())
{
if (storage == storage_for_updating)
break;
if (storage->find(new_entity->getType(), new_entity->getName()))
for (const auto & storage : *storages)
{
throw Exception(
old_entity->formatTypeWithName() + ": cannot rename to " + backQuote(new_entity->getName()) + " because "
+ new_entity->formatTypeWithName() + " already exists in " + storage->getStorageName(),
ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS);
if (storage == storage_for_updating)
break;
if (storage->find(new_entity->getType(), new_entity->getName()))
{
throw Exception(
old_entity->formatTypeWithName() + ": cannot rename to " + backQuote(new_entity->getName()) + " because "
+ new_entity->formatTypeWithName() + " already exists in " + storage->getStorageName(),
ErrorCodes::ACCESS_ENTITY_ALREADY_EXISTS);
}
}
}
}
}
storage_for_updating->update(id, update_func);
return storage_for_updating->update(id, update_func, throw_if_not_exists);
}
@ -275,7 +319,7 @@ scope_guard MultipleAccessStorage::subscribeForChangesImpl(const UUID & id, cons
}
bool MultipleAccessStorage::hasSubscriptionImpl(const UUID & id) const
bool MultipleAccessStorage::hasSubscription(const UUID & id) const
{
auto storages = getStoragesInternal();
for (const auto & storage : *storages)
@ -307,7 +351,7 @@ scope_guard MultipleAccessStorage::subscribeForChangesImpl(AccessEntityType type
}
bool MultipleAccessStorage::hasSubscriptionImpl(AccessEntityType type) const
bool MultipleAccessStorage::hasSubscription(AccessEntityType type) const
{
std::lock_guard lock{mutex};
const auto & handlers = handlers_by_type[static_cast<size_t>(type)];
@ -405,57 +449,24 @@ void MultipleAccessStorage::updateSubscriptionsToNestedStorages(std::unique_lock
}
UUID MultipleAccessStorage::loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const
std::optional<UUID> MultipleAccessStorage::authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists) const
{
auto storages = getStoragesInternal();
for (const auto & storage : *storages)
{
try
auto id = storage->authenticate(credentials, address, external_authenticators, /* throw_if_user_not_exists = */ false);
if (id)
{
auto id = storage->login(credentials, address, external_authenticators, /* replace_exception_with_cannot_authenticate = */ false);
std::lock_guard lock{mutex};
ids_cache.set(id, storage);
ids_cache.set(*id, storage);
return id;
}
catch (...)
{
if (!storage->find(AccessEntityType::USER, credentials.getUserName()))
{
/// The authentication failed because there no users with such name in the `storage`
/// thus we can try to search in other nested storages.
continue;
}
throw;
}
}
throwNotFound(AccessEntityType::USER, credentials.getUserName());
}
UUID MultipleAccessStorage::getIDOfLoggedUserImpl(const String & user_name) const
{
auto storages = getStoragesInternal();
for (const auto & storage : *storages)
{
try
{
auto id = storage->getIDOfLoggedUser(user_name);
std::lock_guard lock{mutex};
ids_cache.set(id, storage);
return id;
}
catch (...)
{
if (!storage->find(AccessEntityType::USER, user_name))
{
/// The authentication failed because there no users with such name in the `storage`
/// thus we can try to search in other nested storages.
continue;
}
throw;
}
}
throwNotFound(AccessEntityType::USER, user_name);
if (throw_if_user_not_exists)
throwNotFound(AccessEntityType::USER, credentials.getUserName());
else
return std::nullopt;
}
}

23
src/Access/MultipleAccessStorage.h
View File

@ -21,6 +21,8 @@ public:
~MultipleAccessStorage() override;
const char * getStorageType() const override { return STORAGE_TYPE; }
bool isReadOnly() const override;
bool isReadOnly(const UUID & id) const override;
void setStorages(const std::vector<StoragePtr> & storages);
void addStorage(const StoragePtr & new_storage);
@ -34,22 +36,21 @@ public:
ConstStoragePtr getStorage(const UUID & id) const;
StoragePtr getStorage(const UUID & id);
bool exists(const UUID & id) const override;
bool hasSubscription(const UUID & id) const override;
bool hasSubscription(AccessEntityType type) const override;
protected:
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
bool existsImpl(const UUID & id) const override;
AccessEntityPtr readImpl(const UUID & id) const override;
String readNameImpl(const UUID &id) const override;
bool canInsertImpl(const AccessEntityPtr & entity) const override;
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(AccessEntityType type) const override;
UUID loginImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators) const override;
UUID getIDOfLoggedUserImpl(const String & user_name) const override;
std::optional<UUID> authenticateImpl(const Credentials & credentials, const Poco::Net::IPAddress & address, const ExternalAuthenticators & external_authenticators, bool throw_if_user_not_exists) const override;
private:
using Storages = std::vector<StoragePtr>;

102
src/Access/ReplicatedAccessStorage.cpp
View File

@ -91,7 +91,7 @@ static void retryOnZooKeeperUserError(size_t attempts, Func && function)
}
}
UUID ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists)
std::optional<UUID> ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, bool replace_if_exists, bool throw_if_exists)
{
const UUID id = generateRandomID();
const AccessEntityTypeInfo type_info = AccessEntityTypeInfo::get(new_entity->getType());
@ -99,7 +99,11 @@ UUID ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, boo
LOG_DEBUG(getLogger(), "Inserting entity of type {} named {} with id {}", type_info.name, name, toString(id));
auto zookeeper = get_zookeeper();
retryOnZooKeeperUserError(10, [&]{ insertZooKeeper(zookeeper, id, new_entity, replace_if_exists); });
bool ok = false;
retryOnZooKeeperUserError(10, [&]{ ok = insertZooKeeper(zookeeper, id, new_entity, replace_if_exists, throw_if_exists); });
if (!ok)
return std::nullopt;
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
@ -109,8 +113,12 @@ UUID ReplicatedAccessStorage::insertImpl(const AccessEntityPtr & new_entity, boo
}
void ReplicatedAccessStorage::insertZooKeeper(
const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & new_entity, bool replace_if_exists)
bool ReplicatedAccessStorage::insertZooKeeper(
const zkutil::ZooKeeperPtr & zookeeper,
const UUID & id,
const AccessEntityPtr & new_entity,
bool replace_if_exists,
bool throw_if_exists)
{
const String & name = new_entity->getName();
const AccessEntityType type = new_entity->getType();
@ -131,6 +139,7 @@ void ReplicatedAccessStorage::insertZooKeeper(
Coordination::Responses responses;
const Coordination::Error res = zookeeper->tryMulti(ops, responses);
if (res == Coordination::Error::ZNODEEXISTS)
{
if (responses[0]->error == Coordination::Error::ZNODEEXISTS)
@ -166,33 +175,47 @@ void ReplicatedAccessStorage::insertZooKeeper(
/// If this fails, then we'll just retry from the start.
zookeeper->multi(replace_ops);
/// Everything's fine, the new entity has been inserted instead of an existing entity.
return true;
}
else
{
throwNameCollisionCannotInsert(type, name);
/// Couldn't insert the new entity because there is an existing entity with such name.
if (throw_if_exists)
throwNameCollisionCannotInsert(type, name);
else
return false;
}
}
else
{
zkutil::KeeperMultiException::check(res, ops, responses);
}
/// If this fails, then we'll just retry from the start.
zkutil::KeeperMultiException::check(res, ops, responses);
/// Everything's fine, the new entity has been inserted.
return true;
}
void ReplicatedAccessStorage::removeImpl(const UUID & id)
bool ReplicatedAccessStorage::removeImpl(const UUID & id, bool throw_if_not_exists)
{
LOG_DEBUG(getLogger(), "Removing entity {}", toString(id));
auto zookeeper = get_zookeeper();
retryOnZooKeeperUserError(10, [&] { removeZooKeeper(zookeeper, id); });
bool ok = false;
retryOnZooKeeperUserError(10, [&] { ok = removeZooKeeper(zookeeper, id, throw_if_not_exists); });
if (!ok)
return false;
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
std::lock_guard lock{mutex};
removeEntityNoLock(id, notifications);
return true;
}
void ReplicatedAccessStorage::removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id)
bool ReplicatedAccessStorage::removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists)
{
const String entity_uuid = toString(id);
const String entity_path = zookeeper_path + "/uuid/" + entity_uuid;
@ -201,7 +224,13 @@ void ReplicatedAccessStorage::removeZooKeeper(const zkutil::ZooKeeperPtr & zooke
Coordination::Stat entity_stat;
const bool uuid_exists = zookeeper->tryGet(entity_path, entity_definition, &entity_stat);
if (!uuid_exists)
throwNotFound(id);
{
/// Couldn't remove, there is no such entity.
if (throw_if_not_exists)
throwNotFound(id);
else
return false;
}
const AccessEntityPtr entity = deserializeAccessEntity(entity_definition, entity_path);
const AccessEntityTypeInfo type_info = AccessEntityTypeInfo::get(entity->getType());
@ -212,26 +241,35 @@ void ReplicatedAccessStorage::removeZooKeeper(const zkutil::ZooKeeperPtr & zooke
Coordination::Requests ops;
ops.emplace_back(zkutil::makeRemoveRequest(entity_path, entity_stat.version));
ops.emplace_back(zkutil::makeRemoveRequest(entity_name_path, -1));
/// If this fails, then we'll just retry from the start.
zookeeper->multi(ops);
/// Everything's fine, the entity has been removed.
return true;
}
void ReplicatedAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func)
bool ReplicatedAccessStorage::updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists)
{
LOG_DEBUG(getLogger(), "Updating entity {}", toString(id));
auto zookeeper = get_zookeeper();
retryOnZooKeeperUserError(10, [&] { updateZooKeeper(zookeeper, id, update_func); });
bool ok = false;
retryOnZooKeeperUserError(10, [&] { ok = updateZooKeeper(zookeeper, id, update_func, throw_if_not_exists); });
if (!ok)
return false;
Notifications notifications;
SCOPE_EXIT({ notify(notifications); });
std::lock_guard lock{mutex};
refreshEntityNoLock(zookeeper, id, notifications);
return true;
}
void ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func)
bool ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists)
{
const String entity_uuid = toString(id);
const String entity_path = zookeeper_path + "/uuid/" + entity_uuid;
@ -240,7 +278,12 @@ void ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zooke
Coordination::Stat stat;
const bool uuid_exists = zookeeper->tryGet(entity_path, old_entity_definition, &stat);
if (!uuid_exists)
throwNotFound(id);
{
if (throw_if_not_exists)
throwNotFound(id);
else
return false;
}
const AccessEntityPtr old_entity = deserializeAccessEntity(old_entity_definition, entity_path);
const AccessEntityPtr new_entity = update_func(old_entity);
@ -276,7 +319,11 @@ void ReplicatedAccessStorage::updateZooKeeper(const zkutil::ZooKeeperPtr & zooke
}
else
{
/// If this fails, then we'll just retry from the start.
zkutil::KeeperMultiException::check(res, ops, responses);
/// Everything's fine, the entity has been updated.
return true;
}
}
@ -525,30 +572,29 @@ std::vector<UUID> ReplicatedAccessStorage::findAllImpl(AccessEntityType type) co
}
bool ReplicatedAccessStorage::existsImpl(const UUID & id) const
bool ReplicatedAccessStorage::exists(const UUID & id) const
{
std::lock_guard lock{mutex};
return entries_by_id.count(id);
}
AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id) const
AccessEntityPtr ReplicatedAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
{
std::lock_guard lock{mutex};
const auto it = entries_by_id.find(id);
if (it == entries_by_id.end())
throwNotFound(id);
{
if (throw_if_not_exists)
throwNotFound(id);
else
return nullptr;
}
const Entry & entry = it->second;
return entry.entity;
}
String ReplicatedAccessStorage::readNameImpl(const UUID & id) const
{
return readImpl(id)->getName();
}
void ReplicatedAccessStorage::prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const
{
const AccessEntityPtr entity = remove ? nullptr : entry.entity;
@ -598,7 +644,7 @@ scope_guard ReplicatedAccessStorage::subscribeForChangesImpl(const UUID & id, co
}
bool ReplicatedAccessStorage::hasSubscriptionImpl(const UUID & id) const
bool ReplicatedAccessStorage::hasSubscription(const UUID & id) const
{
std::lock_guard lock{mutex};
const auto & it = entries_by_id.find(id);
@ -611,7 +657,7 @@ bool ReplicatedAccessStorage::hasSubscriptionImpl(const UUID & id) const
}
bool ReplicatedAccessStorage::hasSubscriptionImpl(AccessEntityType type) const
bool ReplicatedAccessStorage::hasSubscription(AccessEntityType type) const
{
std::lock_guard lock{mutex};
const auto & handlers = handlers_by_type[static_cast<size_t>(type)];

23
src/Access/ReplicatedAccessStorage.h
View File

@ -32,6 +32,10 @@ public:
virtual void startup();
virtual void shutdown();
bool exists(const UUID & id) const override;
bool hasSubscription(const UUID & id) const override;
bool hasSubscription(AccessEntityType type) const override;
private:
String zookeeper_path;
zkutil::GetZooKeeper get_zookeeper;
@ -41,13 +45,13 @@ private:
ThreadFromGlobalPool worker_thread;
ConcurrentBoundedQueue<UUID> refresh_queue;
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
std::optional<UUID> insertImpl(const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists) override;
bool removeImpl(const UUID & id, bool throw_if_not_exists) override;
bool updateImpl(const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists) override;
void insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists);
void removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id);
void updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func);
bool insertZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const AccessEntityPtr & entity, bool replace_if_exists, bool throw_if_exists);
bool removeZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, bool throw_if_not_exists);
bool updateZooKeeper(const zkutil::ZooKeeperPtr & zookeeper, const UUID & id, const UpdateFunc & update_func, bool throw_if_not_exists);
void runWorkerThread();
void resetAfterError();
@ -71,16 +75,11 @@ private:
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
bool existsImpl(const UUID & id) const override;
AccessEntityPtr readImpl(const UUID & id) const override;
String readNameImpl(const UUID & id) const override;
bool canInsertImpl(const AccessEntityPtr &) const override { return true; }
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
void prepareNotifications(const Entry & entry, bool remove, Notifications & notifications) const;
scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(AccessEntityType type) const override;
mutable std::mutex mutex;
std::unordered_map<UUID, Entry> entries_by_id;

34
src/Access/UsersConfigAccessStorage.cpp
View File

@ -610,41 +610,21 @@ std::vector<UUID> UsersConfigAccessStorage::findAllImpl(AccessEntityType type) c
}
bool UsersConfigAccessStorage::existsImpl(const UUID & id) const
bool UsersConfigAccessStorage::exists(const UUID & id) const
{
return memory_storage.exists(id);
}
AccessEntityPtr UsersConfigAccessStorage::readImpl(const UUID & id) const
AccessEntityPtr UsersConfigAccessStorage::readImpl(const UUID & id, bool throw_if_not_exists) const
{
return memory_storage.read(id);
return memory_storage.read(id, throw_if_not_exists);
}
String UsersConfigAccessStorage::readNameImpl(const UUID & id) const
std::optional<String> UsersConfigAccessStorage::readNameImpl(const UUID & id, bool throw_if_not_exists) const
{
return memory_storage.readName(id);
}
UUID UsersConfigAccessStorage::insertImpl(const AccessEntityPtr & entity, bool)
{
throwReadonlyCannotInsert(entity->getType(), entity->getName());
}
void UsersConfigAccessStorage::removeImpl(const UUID & id)
{
auto entity = read(id);
throwReadonlyCannotRemove(entity->getType(), entity->getName());
}
void UsersConfigAccessStorage::updateImpl(const UUID & id, const UpdateFunc &)
{
auto entity = read(id);
throwReadonlyCannotUpdate(entity->getType(), entity->getName());
return memory_storage.readName(id, throw_if_not_exists);
}
@ -660,13 +640,13 @@ scope_guard UsersConfigAccessStorage::subscribeForChangesImpl(AccessEntityType t
}
bool UsersConfigAccessStorage::hasSubscriptionImpl(const UUID & id) const
bool UsersConfigAccessStorage::hasSubscription(const UUID & id) const
{
return memory_storage.hasSubscription(id);
}
bool UsersConfigAccessStorage::hasSubscriptionImpl(AccessEntityType type) const
bool UsersConfigAccessStorage::hasSubscription(AccessEntityType type) const
{
return memory_storage.hasSubscription(type);
}

16
src/Access/UsersConfigAccessStorage.h
View File

@ -27,6 +27,7 @@ public:
const char * getStorageType() const override { return STORAGE_TYPE; }
String getStorageParamsJSON() const override;
bool isReadOnly() const override { return true; }
String getPath() const;
bool isPathEqual(const String & path_) const;
@ -41,22 +42,19 @@ public:
void startPeriodicReloading();
void stopPeriodicReloading();
bool exists(const UUID & id) const override;
bool hasSubscription(const UUID & id) const override;
bool hasSubscription(AccessEntityType type) const override;
private:
void parseFromConfig(const Poco::Util::AbstractConfiguration & config);
std::optional<UUID> findImpl(AccessEntityType type, const String & name) const override;
std::vector<UUID> findAllImpl(AccessEntityType type) const override;
bool existsImpl(const UUID & id) const override;
AccessEntityPtr readImpl(const UUID & id) const override;
String readNameImpl(const UUID & id) const override;
bool canInsertImpl(const AccessEntityPtr &) const override { return false; }
UUID insertImpl(const AccessEntityPtr & entity, bool replace_if_exists) override;
void removeImpl(const UUID & id) override;
void updateImpl(const UUID & id, const UpdateFunc & update_func) override;
AccessEntityPtr readImpl(const UUID & id, bool throw_if_not_exists) const override;
std::optional<String> readNameImpl(const UUID & id, bool throw_if_not_exists) const override;
scope_guard subscribeForChangesImpl(const UUID & id, const OnChangedHandler & handler) const override;
scope_guard subscribeForChangesImpl(AccessEntityType type, const OnChangedHandler & handler) const override;
bool hasSubscriptionImpl(const UUID & id) const override;
bool hasSubscriptionImpl(AccessEntityType type) const override;
MemoryAccessStorage memory_storage;
CheckSettingNameFunction check_setting_name_function;

6
src/AggregateFunctions/AggregateFunctionGroupBitmapData.h
View File

@ -421,6 +421,9 @@ public:
*/
UInt8 rb_contains(UInt64 x) const
{
if (!std::is_same_v<T, UInt64> && x > rb_max())
return 0;
if (isSmall())
return small.find(x) != small.end();
else
@ -432,6 +435,9 @@ public:
*/
void rb_remove(UInt64 x)
{
if (!std::is_same_v<T, UInt64> && x > rb_max())
return;
if (isSmall())
toLarge();

6
src/Client/ClientBase.cpp
View File

@ -8,8 +8,8 @@
#include <unordered_map>
#include <base/argsToConfig.h>
#include <base/DateLUT.h>
#include <base/LocalDate.h>
#include <Common/DateLUT.h>
#include <Common/LocalDate.h>
#include <base/LineReader.h>
#include <base/scope_guard_safe.h>
#include "Common/Exception.h"
@ -561,7 +561,7 @@ void ClientBase::processTextAsSingleQuery(const String & full_query)
try
{
processParsedSingleQuery(full_query, query_to_execute, parsed_query);
processParsedSingleQuery(full_query, query_to_execute, parsed_query, echo_queries);
}
catch (Exception & e)
{

4
src/Client/ClientBaseHelpers.cpp
View File

@ -1,8 +1,8 @@
#include "ClientBaseHelpers.h"
#include <base/DateLUT.h>
#include <base/LocalDate.h>
#include <Common/DateLUT.h>
#include <Common/LocalDate.h>
#include <Parsers/Lexer.h>
#include <Common/UTF8Helpers.h>

36
src/Client/LocalConnection.cpp
View File

@ -214,15 +214,15 @@ bool LocalConnection::poll(size_t)
if (next_packet_type)
return true;
if (send_progress && (state->after_send_progress.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay))
{
state->after_send_progress.restart();
next_packet_type = Protocol::Server::Progress;
return true;
}
if (!state->is_finished)
{
if (send_progress && (state->after_send_progress.elapsedMicroseconds() >= query_context->getSettingsRef().interactive_delay))
{
state->after_send_progress.restart();
next_packet_type = Protocol::Server::Progress;
return true;
}
try
{
pollImpl();
@ -282,6 +282,18 @@ bool LocalConnection::poll(size_t)
}
}
if (state->is_finished && !state->sent_profile_info)
{
state->sent_profile_info = true;
if (state->executor)
{
next_packet_type = Protocol::Server::ProfileInfo;
state->profile_info = state->executor->getProfileInfo();
return true;
}
}
if (state->is_finished)
{
finishQuery();
@ -349,6 +361,16 @@ Packet LocalConnection::receivePacket()
next_packet_type.reset();
break;
}
case Protocol::Server::ProfileInfo:
{
if (state->profile_info)
{
packet.profile_info = std::move(*state->profile_info);
state->profile_info.reset();
}
next_packet_type.reset();
break;
}
case Protocol::Server::TableColumns:
{
if (state->columns_description)

2
src/Client/LocalConnection.h
View File

@ -35,6 +35,7 @@ struct LocalQueryState
/// Current block to be sent next.
std::optional<Block> block;
std::optional<ColumnsDescription> columns_description;
std::optional<ProfileInfo> profile_info;
/// Is request cancelled
bool is_cancelled = false;
@ -43,6 +44,7 @@ struct LocalQueryState
bool sent_totals = false;
bool sent_extremes = false;
bool sent_progress = false;
bool sent_profile_info = false;
/// To output progress, the difference after the previous sending of progress.
Progress progress;

5
src/Columns/ColumnFixedString.h
View File

@ -175,6 +175,11 @@ public:
chars.reserve(n * size);
}
void resize(size_t size)
{
chars.resize(n * size);
}
void getExtremes(Field & min, Field & max) const override;
bool structureEquals(const IColumn & rhs) const override

1
src/Columns/ColumnSparse.cpp
View File

@ -8,6 +8,7 @@
#include <Processors/Transforms/ColumnGathererTransform.h>
#include <algorithm>
#include <bit>
namespace DB
{

91
src/Columns/ColumnStringHelpers.h Normal file
View File

@ -0,0 +1,91 @@
#pragma once
#include <cstring>
#include <cassert>
#include <Columns/IColumn.h>
#include <Common/PODArray.h>
#include <Columns/ColumnString.h>
#include <Columns/ColumnFixedString.h>
#include <IO/WriteBufferFromVector.h>
namespace DB
{
namespace ErrorCodes
{
extern const int TOO_LARGE_STRING_SIZE;
}
namespace ColumnStringHelpers
{
/** Simplifies writing data to the ColumnString or ColumnFixedString via WriteBuffer.
*
* Take care of little subtle details, like padding or proper offsets.
*/
template <typename ColumnType>
class WriteHelper
{
ColumnType & col;
WriteBufferFromVector<typename ColumnType::Chars> buffer;
size_t prev_row_buffer_size = 0;
static ColumnType & resizeColumn(ColumnType & column, size_t rows)
{
if constexpr (std::is_same_v<ColumnType, ColumnFixedString>)
column.resize(rows);
else
{
column.getOffsets().reserve(rows);
/// Using coefficient 2 for initial size is arbitrary.
column.getChars().resize(rows * 2);
}
return column;
}
public:
WriteHelper(ColumnType & col_, size_t expected_rows)
: col(resizeColumn(col_, expected_rows))
, buffer(col.getChars())
{}
~WriteHelper() = default;
void finalize()
{
buffer.finalize();
}
auto & getWriteBuffer()
{
return buffer;
}
inline void rowWritten()
{
if constexpr (std::is_same_v<ColumnType, ColumnFixedString>)
{
if (buffer.count() > prev_row_buffer_size + col.getN())
throw Exception(
ErrorCodes::TOO_LARGE_STRING_SIZE,
"Too large string for FixedString column");
// Pad with zeroes on the right to maintain FixedString invariant.
const auto excess_bytes = buffer.count() % col.getN();
const auto fill_bytes = col.getN() - excess_bytes;
writeChar(0, fill_bytes, buffer);
}
else
{
writeChar(0, buffer);
col.getOffsets().push_back(buffer.count());
}
prev_row_buffer_size = buffer.count();
}
};
}
}

7
src/Common/CMakeLists.txt
View File

@ -1,8 +1,9 @@
add_subdirectory(StringUtils)
# after common_io
#add_subdirectory(ZooKeeper)
#add_subdirectory(ConfigProcessor)
if (ENABLE_EXAMPLES)
add_subdirectory(examples)
endif()
if (USE_MYSQL)
add_subdirectory (mysqlxx)
endif ()

45
src/Common/Config/ConfigProcessor.cpp
View File

@ -18,7 +18,7 @@
#include <Common/ZooKeeper/KeeperException.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/Exception.h>
#include <base/getResource.h>
#include <Common/getResource.h>
#include <base/errnoToString.h>
#include <IO/WriteBufferFromString.h>
#include <IO/Operators.h>
@ -41,24 +41,6 @@ namespace ErrorCodes
/// For cutting preprocessed path to this base
static std::string main_config_path;
/// Extracts from a string the first encountered number consisting of at least two digits.
static std::string numberFromHost(const std::string & s)
{
for (size_t i = 0; i < s.size(); ++i)
{
std::string res;
size_t j = i;
while (j < s.size() && isNumericASCII(s[j]))
res += s[j++];
if (res.size() >= 2)
{
while (res[0] == '0')
res.erase(res.begin());
return res;
}
}
return "";
}
bool ConfigProcessor::isPreprocessedFile(const std::string & path)
{
@ -245,19 +227,6 @@ void ConfigProcessor::merge(XMLDocumentPtr config, XMLDocumentPtr with)
mergeRecursive(config, config_root, with_root);
}
static std::string layerFromHost()
{
struct utsname buf;
if (uname(&buf))
throw Poco::Exception(std::string("uname failed: ") + errnoToString(errno));
std::string layer = numberFromHost(buf.nodename);
if (layer.empty())
throw Poco::Exception(std::string("no layer in host name: ") + buf.nodename);
return layer;
}
void ConfigProcessor::doIncludesRecursive(
XMLDocumentPtr config,
XMLDocumentPtr include_from,
@ -288,18 +257,6 @@ void ConfigProcessor::doIncludesRecursive(
if (node->nodeType() != Node::ELEMENT_NODE)
return;
/// Substitute <layer> for the number extracted from the hostname only if there is an
/// empty <layer> tag without attributes in the original file.
if (node->nodeName() == "layer"
&& !node->hasAttributes()
&& !node->hasChildNodes()
&& node->nodeValue().empty())
{
NodePtr new_node = config->createTextNode(layerFromHost());
node->appendChild(new_node);
return;
}
std::map<std::string, const Node *> attr_nodes;
NamedNodeMapPtr attributes = node->attributes();
size_t substs_count = 0;

1
src/Common/Config/ConfigProcessor.h
View File

@ -59,7 +59,6 @@ public:
/// 4) If zk_node_cache is non-NULL, replace elements matching the "<foo from_zk="/bar">" pattern with
/// "<foo>contents of the /bar ZooKeeper node</foo>".
/// If has_zk_includes is non-NULL and there are such elements, set has_zk_includes to true.
/// 5) (Yandex.Metrika-specific) Substitute "<layer/>" with "<layer>layer number from the hostname</layer>".
XMLDocumentPtr processConfig(
bool * has_zk_includes = nullptr,
zkutil::ZooKeeperNodeCache * zk_node_cache = nullptr,

0
base/base/DateLUT.cpp → src/Common/DateLUT.cpp
View File

2
base/base/DateLUT.h → src/Common/DateLUT.h
View File

@ -2,7 +2,7 @@
#include "DateLUTImpl.h"
#include "defines.h"
#include <base/defines.h>
#include <boost/noncopyable.hpp>

2
base/base/DateLUTImpl.cpp → src/Common/DateLUTImpl.cpp
View File

@ -3,7 +3,7 @@
#include <cctz/civil_time.h>
#include <cctz/time_zone.h>
#include <cctz/zone_info_source.h>
#include <base/getResource.h>
#include <Common/getResource.h>
#include <Poco/Exception.h>
#include <algorithm>

6
base/base/DateLUTImpl.h → src/Common/DateLUTImpl.h
View File

@ -1,8 +1,8 @@
#pragma once
#include "DayNum.h"
#include "defines.h"
#include "types.h"
#include <base/DayNum.h>
#include <base/defines.h>
#include <base/types.h>
#include <ctime>
#include <cassert>

4
src/Common/FiberStack.h
View File

@ -4,7 +4,7 @@
#include <Common/formatReadable.h>
#include <Common/CurrentMemoryTracker.h>
#include <Common/Exception.h>
#include <base/getPageSize.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <sys/mman.h>
@ -36,7 +36,7 @@ public:
explicit FiberStack(size_t stack_size_ = default_stack_size) : stack_size(stack_size_)
{
page_size = ::sysconf(_SC_PAGESIZE);
page_size = getPageSize();
}
boost::context::stack_context allocate()

Some files were not shown because too many files have changed in this diff Show More