Merge branch 'master' into fix-positional

.github/workflows/main.yml

@@ -0,0 +1,78 @@
+name: Ligthweight GithubActions
+on: # yamllint disable-line rule:truthy
+  pull_request:
+    types:
+      - labeled
+      - unlabeled
+      - synchronize
+      - reopened
+      - opened
+    branches:
+      - master
+jobs:
+  CheckLabels:
+    runs-on: [self-hosted]
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Labels check
+        run: cd $GITHUB_WORKSPACE/tests/ci && python3 run_check.py
+        env:
+          ROBOT_TOKEN_0: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN }}
+          ROBOT_TOKEN_1: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_1 }}
+          ROBOT_TOKEN_2: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_2 }}
+          ROBOT_TOKEN_3: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_3 }}
+  DockerHubPush:
+    needs: CheckLabels
+    runs-on: [self-hosted]
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Images check
+        run: cd $GITHUB_WORKSPACE/tests/ci && python3 docker_images_check.py
+        env:
+          YANDEX_S3_ACCESS_KEY_ID: ${{ secrets.YANDEX_S3_ACCESS_KEY_ID }}
+          YANDEX_S3_ACCESS_SECRET_KEY: ${{ secrets.YANDEX_S3_ACCESS_SECRET_KEY }}
+          ROBOT_TOKEN_0: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN }}
+          ROBOT_TOKEN_1: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_1 }}
+          ROBOT_TOKEN_2: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_2 }}
+          ROBOT_TOKEN_3: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_3 }}
+          DOCKER_ROBOT_PASSWORD: ${{ secrets.DOCKER_ROBOT_PASSWORD }}
+      - name: Upload images files to artifacts
+        uses: actions/upload-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/docker_images_check/changed_images.json
+  StyleCheck:
+    needs: DockerHubPush
+    runs-on: [self-hosted]
+    steps:
+      - name: Download changed images
+        uses: actions/download-artifact@v2
+        with:
+          name: changed_images
+          path: ${{ runner.temp }}/style_check
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Style Check
+        env:
+          YANDEX_S3_ACCESS_KEY_ID: ${{ secrets.YANDEX_S3_ACCESS_KEY_ID }}
+          YANDEX_S3_ACCESS_SECRET_KEY: ${{ secrets.YANDEX_S3_ACCESS_SECRET_KEY }}
+          ROBOT_TOKEN_0: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN }}
+          ROBOT_TOKEN_1: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_1 }}
+          ROBOT_TOKEN_2: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_2 }}
+          ROBOT_TOKEN_3: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_3 }}
+        run: cd $GITHUB_WORKSPACE/tests/ci && python3 style_check.py
+  FinishCheck:
+    needs: [StyleCheck, DockerHubPush, CheckLabels]
+    runs-on: [self-hosted]
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+      - name: Finish label
+        run: cd $GITHUB_WORKSPACE/tests/ci && python3 finish_check.py
+        env:
+          ROBOT_TOKEN_0: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN }}
+          ROBOT_TOKEN_1: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_1 }}
+          ROBOT_TOKEN_2: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_2 }}
+          ROBOT_TOKEN_3: ${{ secrets.ROBOT_CLICKHOUSE_PERSONAL_ACCESS_TOKEN_3 }}

cmake/find/capnp.cmake

@@ -34,8 +34,6 @@ endif()
 if (CAPNP_LIBRARIES)
     set (USE_CAPNP 1)
 elseif(NOT MISSING_INTERNAL_CAPNP_LIBRARY)
-    add_subdirectory(contrib/capnproto-cmake)
-
     set (CAPNP_LIBRARIES capnpc)
     set (USE_CAPNP 1)
     set (USE_INTERNAL_CAPNP_LIBRARY 1)

contrib/CMakeLists.txt

@@ -1,16 +1,5 @@
 # Third-party libraries may have substandard code.
 
-# Put all targets defined here and in added subfolders under "contrib/" folder in GUI-based IDEs by default.
-# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they will
-# appear not in "contrib/" as originally planned here.
-get_filename_component (_current_dir_name "${CMAKE_CURRENT_LIST_DIR}" NAME)
-if (CMAKE_FOLDER)
-    set (CMAKE_FOLDER "${CMAKE_FOLDER}/${_current_dir_name}")
-else ()
-    set (CMAKE_FOLDER "${_current_dir_name}")
-endif ()
-unset (_current_dir_name)
-
 set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
 set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
 
@@ -49,6 +38,10 @@ add_subdirectory (replxx-cmake)
 add_subdirectory (unixodbc-cmake)
 add_subdirectory (nanodbc-cmake)
 
+if (USE_INTERNAL_CAPNP_LIBRARY AND NOT MISSING_INTERNAL_CAPNP_LIBRARY)
+    add_subdirectory(capnproto-cmake)
+endif ()
+
 if (ENABLE_FUZZING)
     add_subdirectory (libprotobuf-mutator-cmake)
 endif()
@@ -352,3 +345,76 @@ endif()
 if (USE_S2_GEOMETRY)
     add_subdirectory(s2geometry-cmake)
 endif()
+
+# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
+# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
+# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,
+# instead of controlling it via CMAKE_FOLDER.
+
+function (ensure_target_rooted_in _target _folder)
+    # Skip INTERFACE library targets, since FOLDER property is not available for them.
+    get_target_property (_target_type "${_target}" TYPE)
+    if (_target_type STREQUAL "INTERFACE_LIBRARY")
+        return ()
+    endif ()
+
+    # Read the original FOLDER property value, if any.
+    get_target_property (_folder_prop "${_target}" FOLDER)
+
+    # Normalize that value, so we avoid possible repetitions in folder names.
+
+    if (NOT _folder_prop)
+        set (_folder_prop "")
+    endif ()
+
+    if (CMAKE_FOLDER AND _folder_prop MATCHES "^${CMAKE_FOLDER}/(.*)\$")
+        set (_folder_prop "${CMAKE_MATCH_1}")
+    endif ()
+
+    if (_folder AND _folder_prop MATCHES "^${_folder}/(.*)\$")
+        set (_folder_prop "${CMAKE_MATCH_1}")
+    endif ()
+
+    if (_folder)
+        set (_folder_prop "${_folder}/${_folder_prop}")
+    endif ()
+
+    if (CMAKE_FOLDER)
+        set (_folder_prop "${CMAKE_FOLDER}/${_folder_prop}")
+    endif ()
+
+    # Set the updated FOLDER property value back.
+    set_target_properties ("${_target}" PROPERTIES FOLDER "${_folder_prop}")
+endfunction ()
+
+function (ensure_own_targets_are_rooted_in _dir _folder)
+    get_directory_property (_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS)
+    foreach (_target IN LISTS _targets)
+        ensure_target_rooted_in ("${_target}" "${_folder}")
+    endforeach ()
+endfunction ()
+
+function (ensure_all_targets_are_rooted_in _dir _folder)
+    ensure_own_targets_are_rooted_in ("${_dir}" "${_folder}")
+
+    get_property (_sub_dirs DIRECTORY "${_dir}" PROPERTY SUBDIRECTORIES)
+    foreach (_sub_dir IN LISTS _sub_dirs)
+        ensure_all_targets_are_rooted_in ("${_sub_dir}" "${_folder}")
+    endforeach ()
+endfunction ()
+
+function (organize_ide_folders_2_level _dir)
+    get_filename_component (_dir_name "${_dir}" NAME)
+    ensure_own_targets_are_rooted_in ("${_dir}" "${_dir_name}")
+
+    # Note, that we respect only first two levels of nesting, we don't want to
+    # reorganize target folders further within each third-party dir.
+
+    get_property (_sub_dirs DIRECTORY "${_dir}" PROPERTY SUBDIRECTORIES)
+    foreach (_sub_dir IN LISTS _sub_dirs)
+        get_filename_component (_sub_dir_name "${_sub_dir}" NAME)
+        ensure_all_targets_are_rooted_in ("${_sub_dir}" "${_dir_name}/${_sub_dir_name}")
+    endforeach ()
+endfunction ()
+
+organize_ide_folders_2_level ("${CMAKE_CURRENT_LIST_DIR}")

contrib/libcxx-cmake/CMakeLists.txt

@@ -47,6 +47,7 @@ set(SRCS
 )
 
 add_library(cxx ${SRCS})
+set_target_properties(cxx PROPERTIES FOLDER "contrib/libcxx-cmake")
 
 target_include_directories(cxx SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBCXX_SOURCE_DIR}/include>)
 target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)

contrib/libcxxabi-cmake/CMakeLists.txt

@@ -22,6 +22,7 @@ set(SRCS
 )
 
 add_library(cxxabi ${SRCS})
+set_target_properties(cxxabi PROPERTIES FOLDER "contrib/libcxxabi-cmake")
 
 # Third party library may have substandard code.
 target_compile_options(cxxabi PRIVATE -w)

contrib/libunwind-cmake/CMakeLists.txt

@@ -39,6 +39,7 @@ set(LIBUNWIND_SOURCES
     ${LIBUNWIND_ASM_SOURCES})
 
 add_library(unwind ${LIBUNWIND_SOURCES})
+set_target_properties(unwind PROPERTIES FOLDER "contrib/libunwind-cmake")
 
 target_include_directories(unwind SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBUNWIND_SOURCE_DIR}/include>)
 target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)

docker/test/integration/kerberized_hadoop/Dockerfile

@@ -1,16 +1,22 @@
 # docker build -t clickhouse/kerberized-hadoop .
 
 FROM sequenceiq/hadoop-docker:2.7.0
-RUN sed -i -e 's/^\#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo
-RUN sed -i -e 's/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Base.repo
-RUN sed -i -e 's#http://mirror.centos.org/#http://vault.centos.org/#' /etc/yum.repos.d/CentOS-Base.repo
+
+RUN sed -i -e 's/^\#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo && \
+	sed -i -e 's/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Base.repo && \
+	sed -i -e 's#http://mirror.centos.org/#http://vault.centos.org/#' /etc/yum.repos.d/CentOS-Base.repo
+
+# https://community.letsencrypt.org/t/rhel-centos-6-openssl-client-compatibility-after-dst-root-ca-x3-expiration/161032/81
+RUN sed -i s/xMDkzMDE0MDExNVow/0MDkzMDE4MTQwM1ow/ /etc/pki/tls/certs/ca-bundle.crt
+
 RUN yum clean all && \
 	rpm --rebuilddb &&	\
 	yum -y update && \
 	yum -y install yum-plugin-ovl && \
 	yum --quiet -y install krb5-workstation.x86_64
+
 RUN cd /tmp && \
-	curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz   -o  commons-daemon-1.0.15-src.tar.gz && \
+	curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \
 	tar xzf commons-daemon-1.0.15-src.tar.gz && \
 	cd commons-daemon-1.0.15-src/src/native/unix && \
 	./configure && \

docker/test/pvs/Dockerfile

@@ -37,7 +37,9 @@ RUN set -x \
     || echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \
     && dpkg -i "${PKG_VERSION}.deb"
 
-CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic  \
+ENV CCACHE_DIR=/test_output/ccache
+
+CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic  \
     && cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
     && ninja re2_st clickhouse_grpc_protos \
     && pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \

docker/test/style/run.sh

@@ -1,5 +1,7 @@
 #!/bin/bash
 
+# yaml check is not the best one
+
 cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv
 ./check-style -n              |& tee /test_output/style_output.txt
 ./check-typos                 |& tee /test_output/typos_output.txt

src/Common/examples/CMakeLists.txt

@@ -77,3 +77,6 @@ target_link_libraries (average PRIVATE clickhouse_common_io)
 
 add_executable (shell_command_inout shell_command_inout.cpp)
 target_link_libraries (shell_command_inout PRIVATE clickhouse_common_io)
+
+add_executable (executable_udf executable_udf.cpp)
+target_link_libraries (executable_udf PRIVATE dbms)

src/Common/examples/executable_udf.cpp

@@ -0,0 +1,44 @@
+#include <vector>
+#include <string>
+#include <iomanip>
+
+#include <Common/SipHash.h>
+
+#include <IO/ReadBufferFromFileDescriptor.h>
+#include <IO/WriteBufferFromFileDescriptor.h>
+#include <IO/ReadHelpers.h>
+#include <IO/WriteHelpers.h>
+#include <Common/Stopwatch.h>
+
+using namespace DB;
+
+int main(int argc, char **argv)
+{
+    (void)(argc);
+    (void)(argv);
+
+    std::string buffer;
+
+    ReadBufferFromFileDescriptor read_buffer(0);
+    WriteBufferFromFileDescriptor write_buffer(1);
+    size_t rows = 0;
+    char dummy;
+
+    while (!read_buffer.eof()) {
+        readIntText(rows, read_buffer);
+        readChar(dummy, read_buffer);
+
+        for (size_t i = 0; i < rows; ++i) {
+            readString(buffer, read_buffer);
+            readChar(dummy, read_buffer);
+
+            writeString("Key ", write_buffer);
+            writeString(buffer, write_buffer);
+            writeChar('\n', write_buffer);
+        }
+
+        write_buffer.next();
+    }
+
+    return 0;
+}

src/Common/filesystemHelpers.cpp

@@ -118,7 +118,7 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p
     return absolute_path.starts_with(absolute_prefix_path);
 }
 
-bool symlinkStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path)
+bool fileOrSymlinkPathStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path)
 {
     /// Differs from pathStartsWith in how `path` is normalized before comparison.
     /// Make `path` absolute if it was relative and put it into normalized form: remove
@@ -140,13 +140,14 @@ bool pathStartsWith(const String & path, const String & prefix_path)
     return pathStartsWith(filesystem_path, filesystem_prefix_path);
 }
 
-bool symlinkStartsWith(const String & path, const String & prefix_path)
+bool fileOrSymlinkPathStartsWith(const String & path, const String & prefix_path)
 {
     auto filesystem_path = std::filesystem::path(path);
     auto filesystem_prefix_path = std::filesystem::path(prefix_path);
 
-    return symlinkStartsWith(filesystem_path, filesystem_prefix_path);
+    return fileOrSymlinkPathStartsWith(filesystem_path, filesystem_prefix_path);
 }
+
 }
 
 

src/Common/filesystemHelpers.h

@@ -35,8 +35,9 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p
 /// Returns true if path starts with prefix path
 bool pathStartsWith(const String & path, const String & prefix_path);
 
-/// Returns true if symlink starts with prefix path
-bool symlinkStartsWith(const String & path, const String & prefix_path);
+/// Same as pathStartsWith, but without canonization, i.e. allowed to check symlinks.
+/// (Path is made absolute and normalized.)
+bool fileOrSymlinkPathStartsWith(const String & path, const String & prefix_path);
 
 }
 

src/Dictionaries/FileDictionarySource.cpp

@@ -31,7 +31,7 @@ FileDictionarySource::FileDictionarySource(
     , context(context_)
 {
     auto user_files_path = context->getUserFilesPath();
-    if (created_from_ddl && !pathStartsWith(filepath, user_files_path))
+    if (created_from_ddl && !fileOrSymlinkPathStartsWith(filepath, user_files_path))
         throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", filepath, user_files_path);
 }
 

src/Dictionaries/LibraryDictionarySource.cpp

@@ -41,13 +41,7 @@ LibraryDictionarySource::LibraryDictionarySource(
     , context(Context::createCopy(context_))
 {
     auto dictionaries_lib_path = context->getDictionariesLibPath();
-    bool path_checked = false;
-    if (fs::is_symlink(path))
-        path_checked = symlinkStartsWith(path, dictionaries_lib_path);
-    else
-        path_checked = pathStartsWith(path, dictionaries_lib_path);
-
-    if (created_from_ddl && !path_checked)
+    if (created_from_ddl && !fileOrSymlinkPathStartsWith(path, dictionaries_lib_path))
         throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, dictionaries_lib_path);
 
     if (!fs::exists(path))

src/Functions/readWkt.cpp

@@ -18,10 +18,10 @@ namespace ErrorCodes
 
 
 template <class DataTypeName, class Geometry, class Serializer, class NameHolder>
-class FunctionReadWkt : public IFunction
+class FunctionReadWKT : public IFunction
 {
 public:
-    explicit FunctionReadWkt() = default;
+    explicit FunctionReadWKT() = default;
 
     static constexpr const char * name = NameHolder::name;
 
@@ -72,36 +72,36 @@ public:
 
     static FunctionPtr create(ContextPtr)
     {
-        return std::make_shared<FunctionReadWkt<DataTypeName, Geometry, Serializer, NameHolder>>();
+        return std::make_shared<FunctionReadWKT<DataTypeName, Geometry, Serializer, NameHolder>>();
     }
 };
 
-struct ReadWktPointNameHolder
+struct ReadWKTPointNameHolder
 {
-    static constexpr const char * name = "readWktPoint";
+    static constexpr const char * name = "readWKTPoint";
 };
 
-struct ReadWktRingNameHolder
+struct ReadWKTRingNameHolder
 {
-    static constexpr const char * name = "readWktRing";
+    static constexpr const char * name = "readWKTRing";
 };
 
-struct ReadWktPolygonNameHolder
+struct ReadWKTPolygonNameHolder
 {
-    static constexpr const char * name = "readWktPolygon";
+    static constexpr const char * name = "readWKTPolygon";
 };
 
-struct ReadWktMultiPolygonNameHolder
+struct ReadWKTMultiPolygonNameHolder
 {
-    static constexpr const char * name = "readWktMultiPolygon";
+    static constexpr const char * name = "readWKTMultiPolygon";
 };
 
-void registerFunctionReadWkt(FunctionFactory & factory)
+void registerFunctionReadWKT(FunctionFactory & factory)
 {
-    factory.registerFunction<FunctionReadWkt<DataTypePointName, CartesianPoint, PointSerializer<CartesianPoint>, ReadWktPointNameHolder>>();
-    factory.registerFunction<FunctionReadWkt<DataTypeRingName, CartesianRing, RingSerializer<CartesianPoint>, ReadWktRingNameHolder>>();
-    factory.registerFunction<FunctionReadWkt<DataTypePolygonName, CartesianPolygon, PolygonSerializer<CartesianPoint>, ReadWktPolygonNameHolder>>();
-    factory.registerFunction<FunctionReadWkt<DataTypeMultiPolygonName, CartesianMultiPolygon, MultiPolygonSerializer<CartesianPoint>, ReadWktMultiPolygonNameHolder>>();
+    factory.registerFunction<FunctionReadWKT<DataTypePointName, CartesianPoint, PointSerializer<CartesianPoint>, ReadWKTPointNameHolder>>();
+    factory.registerFunction<FunctionReadWKT<DataTypeRingName, CartesianRing, RingSerializer<CartesianPoint>, ReadWKTRingNameHolder>>();
+    factory.registerFunction<FunctionReadWKT<DataTypePolygonName, CartesianPolygon, PolygonSerializer<CartesianPoint>, ReadWKTPolygonNameHolder>>();
+    factory.registerFunction<FunctionReadWKT<DataTypeMultiPolygonName, CartesianMultiPolygon, MultiPolygonSerializer<CartesianPoint>, ReadWKTMultiPolygonNameHolder>>();
 }
 
 }

src/Functions/registerFunctionsGeo.cpp

@@ -23,7 +23,7 @@ void registerFunctionGeohashEncode(FunctionFactory & factory);
 void registerFunctionGeohashDecode(FunctionFactory & factory);
 void registerFunctionGeohashesInBox(FunctionFactory & factory);
 void registerFunctionWkt(FunctionFactory & factory);
-void registerFunctionReadWkt(FunctionFactory & factory);
+void registerFunctionReadWKT(FunctionFactory & factory);
 void registerFunctionSvg(FunctionFactory & factory);
 
 #if USE_H3
@@ -79,7 +79,7 @@ void registerFunctionsGeo(FunctionFactory & factory)
     registerFunctionGeohashDecode(factory);
     registerFunctionGeohashesInBox(factory);
     registerFunctionWkt(factory);
-    registerFunctionReadWkt(factory);
+    registerFunctionReadWKT(factory);
     registerFunctionSvg(factory);
 
 #if USE_H3

src/Functions/svg.cpp

@@ -102,6 +102,7 @@ public:
 void registerFunctionSvg(FunctionFactory & factory)
 {
     factory.registerFunction<FunctionSvg>();
+    factory.registerAlias("SVG", "svg");
 }
 
 }

src/Storages/StorageFile.cpp

@@ -22,6 +22,7 @@
 #include <Common/escapeForFileName.h>
 #include <Common/typeid_cast.h>
 #include <Common/parseGlobs.h>
+#include <Common/filesystemHelpers.h>
 #include <Storages/ColumnsDescription.h>
 #include <Storages/StorageInMemoryMetadata.h>
 
@@ -124,8 +125,8 @@ void checkCreationIsAllowed(ContextPtr context_global, const std::string & db_di
         return;
 
     /// "/dev/null" is allowed for perf testing
-    if (!startsWith(table_path, db_dir_path) && table_path != "/dev/null")
-        throw Exception("File is not inside " + db_dir_path, ErrorCodes::DATABASE_ACCESS_DENIED);
+    if (!fileOrSymlinkPathStartsWith(table_path, db_dir_path) && table_path != "/dev/null")
+        throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File `{}` is not inside `{}`", table_path, db_dir_path);
 
     if (fs::exists(table_path) && fs::is_directory(table_path))
         throw Exception("File must not be a directory", ErrorCodes::INCORRECT_FILE_NAME);
@@ -140,7 +141,10 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user
         fs_table_path = user_files_absolute_path / fs_table_path;
 
     Strings paths;
-    const String path = fs::weakly_canonical(fs_table_path);
+    /// Do not use fs::canonical or fs::weakly_canonical.
+    /// Otherwise it will not allow to work with symlinks in `user_files_path` directory.
+    String path = fs::absolute(fs_table_path);
+    path = fs::path(path).lexically_normal(); /// Normalize path.
     if (path.find_first_of("*?{") == std::string::npos)
     {
         std::error_code error;

tests/ci/compress_files.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+import subprocess
+import logging
+import os
+
+def compress_file_fast(path, archive_path):
+    if os.path.exists('/usr/bin/pigz'):
+        subprocess.check_call("pigz < {} > {}".format(path, archive_path), shell=True)
+    else:
+        subprocess.check_call("gzip < {} > {}".format(path, archive_path), shell=True)
+
+
+def compress_fast(path, archive_path, exclude=None):
+    pigz_part = ''
+    if os.path.exists('/usr/bin/pigz'):
+        logging.info("pigz found, will compress and decompress faster")
+        pigz_part = "--use-compress-program='pigz'"
+    else:
+        pigz_part = '-z'
+        logging.info("no pigz, compressing with default tar")
+
+    if exclude is None:
+        exclude_part = ""
+    elif isinstance(exclude, list):
+        exclude_part = " ".join(["--exclude {}".format(x) for x in exclude])
+    else:
+        exclude_part = "--exclude {}".format(str(exclude))
+
+    fname = os.path.basename(path)
+    if os.path.isfile(path):
+        path = os.path.dirname(path)
+    else:
+        path += "/.."
+    cmd = "tar {} {} -cf {} -C {} {}".format(pigz_part, exclude_part, archive_path, path, fname)
+    logging.debug("compress_fast cmd:{}".format(cmd))
+    subprocess.check_call(cmd, shell=True)
+
+
+def decompress_fast(archive_path, result_path=None):
+    pigz_part = ''
+    if os.path.exists('/usr/bin/pigz'):
+        logging.info("pigz found, will compress and decompress faster ('{}' -> '{}')".format(archive_path, result_path))
+        pigz_part = "--use-compress-program='pigz'"
+    else:
+        pigz_part = '-z'
+        logging.info("no pigz, decompressing with default tar ('{}' -> '{}')".format(archive_path, result_path))
+
+    if result_path is None:
+        subprocess.check_call("tar {} -xf {}".format(pigz_part, archive_path), shell=True)
+    else:
+        subprocess.check_call("tar {} -xf {} -C {}".format(pigz_part, archive_path, result_path), shell=True)

tests/ci/docker_images_check.py

@@ -0,0 +1,234 @@
+#!/usr/bin/env python3
+import subprocess
+import logging
+from report import create_test_html_report
+from s3_helper import S3Helper
+import json
+import os
+from pr_info import PRInfo
+from github import Github
+import shutil
+from get_robot_token import get_best_robot_token
+
+NAME = "Push to Dockerhub (actions)"
+
+def get_changed_docker_images(pr_info, repo_path, image_file_path):
+    images_dict = {}
+    path_to_images_file = os.path.join(repo_path, image_file_path)
+    if os.path.exists(path_to_images_file):
+        with open(path_to_images_file, 'r') as dict_file:
+            images_dict = json.load(dict_file)
+    else:
+        logging.info("Image file %s doesnt exists in repo %s", image_file_path, repo_path)
+
+    dockerhub_repo_name = 'yandex'
+    if not images_dict:
+        return [], dockerhub_repo_name
+
+    files_changed = pr_info.changed_files
+
+    logging.info("Changed files for PR %s @ %s: %s", pr_info.number, pr_info.sha, str(files_changed))
+
+    changed_images = []
+
+    for dockerfile_dir, image_description in images_dict.items():
+        if image_description['name'].startswith('clickhouse/'):
+            dockerhub_repo_name = 'clickhouse'
+
+        for f in files_changed:
+            if f.startswith(dockerfile_dir):
+                logging.info(
+                    "Found changed file '%s' which affects docker image '%s' with path '%s'",
+                    f, image_description['name'], dockerfile_dir)
+                changed_images.append(dockerfile_dir)
+                break
+
+    # The order is important: dependents should go later than bases, so that
+    # they are built with updated base versions.
+    index = 0
+    while index < len(changed_images):
+        image = changed_images[index]
+        for dependent in images_dict[image]['dependent']:
+            logging.info(
+                "Marking docker image '%s' as changed because it depends on changed docker image '%s'",
+                dependent, image)
+            changed_images.append(dependent)
+        index += 1
+        if index > 100:
+            # Sanity check to prevent infinite loop.
+            raise "Too many changed docker images, this is a bug." + str(changed_images)
+
+    # If a dependent image was already in the list because its own files
+    # changed, but then it was added as a dependent of a changed base, we
+    # must remove the earlier entry so that it doesn't go earlier than its
+    # base. This way, the dependent will be rebuilt later than the base, and
+    # will correctly use the updated version of the base.
+    seen = set()
+    no_dups_reversed = []
+    for x in reversed(changed_images):
+        if x not in seen:
+            seen.add(x)
+            no_dups_reversed.append(x)
+
+    result = [(x, images_dict[x]['name']) for x in reversed(no_dups_reversed)]
+    logging.info("Changed docker images for PR %s @ %s: '%s'", pr_info.number, pr_info.sha, result)
+    return result, dockerhub_repo_name
+
+def build_and_push_one_image(path_to_dockerfile_folder, image_name, version_string):
+    logging.info("Building docker image %s with version %s from path %s", image_name, version_string, path_to_dockerfile_folder)
+    build_log = None
+    push_log = None
+    with open('build_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
+        cmd = "docker build --network=host -t {im}:{ver} {path}".format(im=image_name, ver=version_string, path=path_to_dockerfile_folder)
+        retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
+        build_log = str(pl.name)
+        if retcode != 0:
+            return False, build_log, None
+
+    with open('tag_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
+        cmd = "docker build --network=host -t {im} {path}".format(im=image_name, path=path_to_dockerfile_folder)
+        retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
+        build_log = str(pl.name)
+        if retcode != 0:
+            return False, build_log, None
+
+    logging.info("Pushing image %s to dockerhub", image_name)
+
+    with open('push_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
+        cmd = "docker push {im}:{ver}".format(im=image_name, ver=version_string)
+        retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
+        push_log = str(pl.name)
+        if retcode != 0:
+            return False, build_log, push_log
+
+    logging.info("Processing of %s successfully finished", image_name)
+    return True, build_log, push_log
+
+def process_single_image(versions, path_to_dockerfile_folder, image_name):
+    logging.info("Image will be pushed with versions %s", ', '.join(versions))
+    result = []
+    for ver in versions:
+        for i in range(5):
+            success, build_log, push_log = build_and_push_one_image(path_to_dockerfile_folder, image_name, ver)
+            if success:
+                result.append((image_name + ":" + ver, build_log, push_log, 'OK'))
+                break
+            logging.info("Got error will retry %s time and sleep for %s seconds", i, i * 5)
+            time.sleep(i * 5)
+        else:
+            result.append((image_name + ":" + ver, build_log, push_log, 'FAIL'))
+
+    logging.info("Processing finished")
+    return result
+
+
+def process_test_results(s3_client, test_results, s3_path_prefix):
+    overall_status = 'success'
+    processed_test_results = []
+    for image, build_log, push_log, status in test_results:
+        if status != 'OK':
+            overall_status = 'failure'
+        url_part = ''
+        if build_log is not None and os.path.exists(build_log):
+            build_url = s3_client.upload_test_report_to_s3(
+                build_log,
+                s3_path_prefix + "/" + os.path.basename(build_log))
+            url_part += '<a href="{}">build_log</a>'.format(build_url)
+        if push_log is not None and os.path.exists(push_log):
+            push_url = s3_client.upload_test_report_to_s3(
+                push_log,
+                s3_path_prefix + "/" + os.path.basename(push_log))
+            if url_part:
+                url_part += ', '
+            url_part += '<a href="{}">push_log</a>'.format(push_url)
+        if url_part:
+            test_name = image + ' (' + url_part + ')'
+        else:
+            test_name = image
+        processed_test_results.append((test_name, status))
+    return overall_status, processed_test_results
+
+def upload_results(s3_client, pr_number, commit_sha, test_results):
+    s3_path_prefix = f"{pr_number}/{commit_sha}/" + NAME.lower().replace(' ', '_')
+
+    branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
+    branch_name = "master"
+    if pr_number != 0:
+        branch_name = "PR #{}".format(pr_number)
+        branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
+    commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
+
+    task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
+
+    html_report = create_test_html_report(NAME, test_results, "https://hub.docker.com/u/clickhouse", task_url, branch_url, branch_name, commit_url)
+    with open('report.html', 'w') as f:
+        f.write(html_report)
+
+    url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
+    logging.info("Search result in url %s", url)
+    return url
+
+def get_commit(gh, commit_sha):
+    repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
+    commit = repo.get_commit(commit_sha)
+    return commit
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    repo_path = os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../"))
+    temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'docker_images_check')
+    dockerhub_password = os.getenv('DOCKER_ROBOT_PASSWORD')
+
+    if os.path.exists(temp_path):
+        shutil.rmtree(temp_path)
+
+    if not os.path.exists(temp_path):
+        os.makedirs(temp_path)
+
+    with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
+        event = json.load(event_file)
+
+    pr_info = PRInfo(event, False, True)
+    changed_images, dockerhub_repo_name = get_changed_docker_images(pr_info, repo_path, "docker/images.json")
+    logging.info("Has changed images %s", ', '.join([str(image[0]) for image in changed_images]))
+    pr_commit_version = str(pr_info.number) + '-' + pr_info.sha
+
+    versions = [str(pr_info.number), pr_commit_version]
+
+    subprocess.check_output("docker login --username 'robotclickhouse' --password '{}'".format(dockerhub_password), shell=True)
+
+    result_images = {}
+    images_processing_result = []
+    for rel_path, image_name in changed_images:
+        full_path = os.path.join(repo_path, rel_path)
+        images_processing_result += process_single_image(versions, full_path, image_name)
+        result_images[image_name] = pr_commit_version
+
+    if len(changed_images):
+        description = "Updated " + ','.join([im[1] for im in changed_images])
+    else:
+        description = "Nothing to update"
+
+
+    if len(description) >= 140:
+        description = description[:136] + "..."
+
+    aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
+    aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
+
+    s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
+
+    s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_')
+    status, test_results = process_test_results(s3_helper, images_processing_result, s3_path_prefix)
+
+    url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results)
+
+    gh = Github(get_best_robot_token())
+    commit = get_commit(gh, pr_info.sha)
+    commit.create_status(context=NAME, description=description, state=status, target_url=url)
+
+    with open(os.path.join(temp_path, 'changed_images.json'), 'w') as images_file:
+        json.dump(result_images, images_file)
+
+    print("::notice ::Report url: {}".format(url))
+    print("::set-output name=url_output::\"{}\"".format(url))

tests/ci/finish_check.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+import logging
+from github import Github
+from pr_info import PRInfo
+import json
+import os
+from get_robot_token import get_best_robot_token
+
+NAME = 'Run Check (actions)'
+
+def filter_statuses(statuses):
+    """
+    Squash statuses to latest state
+    1. context="first", state="success", update_time=1
+    2. context="second", state="success", update_time=2
+    3. context="first", stat="failure", update_time=3
+    =========>
+    1. context="second", state="success"
+    2. context="first", stat="failure"
+    """
+    filt = {}
+    for status in sorted(statuses, key=lambda x: x.updated_at):
+        filt[status.context] = status
+    return filt
+
+
+def get_commit(gh, commit_sha):
+    repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
+    commit = repo.get_commit(commit_sha)
+    return commit
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
+        event = json.load(event_file)
+
+    pr_info = PRInfo(event, need_orgs=True)
+    gh = Github(get_best_robot_token())
+    commit = get_commit(gh, pr_info.sha)
+
+    url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
+    statuses = filter_statuses(list(commit.get_statuses()))
+    if NAME in statuses and statuses[NAME].state == "pending":
+        commit.create_status(context=NAME, description="All checks finished", state="success", target_url=url)

tests/ci/get_robot_token.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+import os
+from github import Github
+
+def get_best_robot_token(token_prefix_env_name="ROBOT_TOKEN_", total_tokens=4):
+    tokens = {}
+    for i in range(total_tokens):
+        token_name = token_prefix_env_name + str(i)
+        token = os.getenv(token_name)
+        gh = Github(token)
+        rest, _ = gh.rate_limiting
+        tokens[token] = rest
+
+    return max(tokens.items(), key=lambda x: x[1])[0]

tests/ci/metrics_lambda/Dockerfile

@@ -0,0 +1,13 @@
+FROM public.ecr.aws/lambda/python:3.9
+
+# Copy function code
+COPY app.py ${LAMBDA_TASK_ROOT}
+
+# Install the function's dependencies using file requirements.txt
+# from your project folder.
+
+COPY requirements.txt  .
+RUN  pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "app.handler" ]

tests/ci/metrics_lambda/app.py

@@ -0,0 +1,143 @@
+#!/usr/bin/env python3
+
+import requests
+import argparse
+import jwt
+import sys
+import json
+import time
+from collections import namedtuple
+
+def get_key_and_app_from_aws():
+    import boto3
+    secret_name = "clickhouse_github_secret_key"
+    session = boto3.session.Session()
+    client = session.client(
+        service_name='secretsmanager',
+    )
+    get_secret_value_response = client.get_secret_value(
+        SecretId=secret_name
+    )
+    data = json.loads(get_secret_value_response['SecretString'])
+    return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
+
+def handler(event, context):
+    private_key, app_id = get_key_and_app_from_aws()
+    main(private_key, app_id, True)
+
+def get_installation_id(jwt_token):
+    headers = {
+        "Authorization": f"Bearer {jwt_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+    response = requests.get("https://api.github.com/app/installations", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    return data[0]['id']
+
+def get_access_token(jwt_token, installation_id):
+    headers = {
+        "Authorization": f"Bearer {jwt_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+    response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    return data['token']
+
+
+RunnerDescription = namedtuple('RunnerDescription', ['id', 'name', 'tags', 'offline', 'busy'])
+
+def list_runners(access_token):
+    headers = {
+        "Authorization": f"token {access_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+
+    response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    print("Total runners", data['total_count'])
+    runners = data['runners']
+    result = []
+    for runner in runners:
+        tags = [tag['name'] for tag in runner['labels']]
+        desc = RunnerDescription(id=runner['id'], name=runner['name'], tags=tags,
+                                 offline=runner['status']=='offline', busy=runner['busy'])
+        result.append(desc)
+    return result
+
+def push_metrics_to_cloudwatch(listed_runners, namespace):
+    import boto3
+    client = boto3.client('cloudwatch')
+    metrics_data = []
+    busy_runners = sum(1 for runner in listed_runners if runner.busy)
+    metrics_data.append({
+        'MetricName': 'BusyRunners',
+        'Value': busy_runners,
+        'Unit': 'Count',
+    })
+    total_active_runners = sum(1 for runner in listed_runners if not runner.offline)
+    metrics_data.append({
+        'MetricName': 'ActiveRunners',
+        'Value': total_active_runners,
+        'Unit': 'Count',
+    })
+    total_runners = len(listed_runners)
+    metrics_data.append({
+        'MetricName': 'TotalRunners',
+        'Value': total_runners,
+        'Unit': 'Count',
+    })
+    if total_active_runners == 0:
+        busy_ratio = 100
+    else:
+        busy_ratio = busy_runners / total_active_runners * 100
+
+    metrics_data.append({
+        'MetricName': 'BusyRunnersRatio',
+        'Value': busy_ratio,
+        'Unit': 'Percent',
+    })
+
+    client.put_metric_data(Namespace='RunnersMetrics', MetricData=metrics_data)
+
+def main(github_secret_key, github_app_id, push_to_cloudwatch):
+    payload = {
+        "iat": int(time.time()) - 60,
+        "exp": int(time.time()) + (10 * 60),
+        "iss": github_app_id,
+    }
+
+    encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
+    installation_id = get_installation_id(encoded_jwt)
+    access_token = get_access_token(encoded_jwt, installation_id)
+    runners = list_runners(access_token)
+    if push_to_cloudwatch:
+        push_metrics_to_cloudwatch(runners, 'RunnersMetrics')
+    else:
+        print(runners)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Get list of runners and their states')
+    parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
+    parser.add_argument('-k', '--private-key', help='Private key')
+    parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
+    parser.add_argument('--push-to-cloudwatch', action='store_true',  help='Store received token in parameter store')
+
+    args = parser.parse_args()
+
+    if not args.private_key_path and not args.private_key:
+        print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
+
+    if args.private_key_path and args.private_key:
+        print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
+
+    if args.private_key:
+        private_key = args.private_key
+    else:
+        with open(args.private_key_path, 'r') as key_file:
+            private_key = key_file.read()
+
+    main(private_key, args.app_id, args.push_to_cloudwatch)

tests/ci/metrics_lambda/requirements.txt

@@ -0,0 +1,3 @@
+requests
+PyJWT
+cryptography

tests/ci/pr_info.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python3
+import requests
+import json
+import os
+import subprocess
+import urllib
+from unidiff import PatchSet
+
+
+class PRInfo:
+    def __init__(self, github_event, need_orgs=False, need_changed_files=False):
+        self.number = github_event['number']
+        if 'after' in github_event:
+            self.sha = github_event['after']
+        else:
+            self.sha = github_event['pull_request']['head']['sha']
+
+        self.labels = set([l['name'] for l in github_event['pull_request']['labels']])
+        self.user_login = github_event['pull_request']['user']['login']
+        self.user_orgs = set([])
+        if need_orgs:
+            user_orgs_response = requests.get(github_event['pull_request']['user']['organizations_url'])
+            if user_orgs_response.ok:
+                response_json = user_orgs_response.json()
+                self.user_orgs = set(org['id'] for org in response_json)
+
+        self.changed_files = set([])
+        if need_changed_files:
+            diff_url = github_event['pull_request']['diff_url']
+            diff = urllib.request.urlopen(github_event['pull_request']['diff_url'])
+            diff_object = PatchSet(diff, diff.headers.get_charsets()[0])
+            self.changed_files = set([f.path for f in diff_object])
+
+    def get_dict(self):
+        return {
+            'sha': self.sha,
+            'number': self.number,
+            'labels': self.labels,
+            'user_login': self.user_login,
+            'user_orgs': self.user_orgs,
+        }

tests/ci/pvs_check.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+import subprocess
+import os
+import json
+import logging
+from github import Github
+from report import create_test_html_report
+from s3_helper import S3Helper
+from pr_info import PRInfo
+import shutil
+import sys
+from get_robot_token import get_best_robot_token
+
+NAME = 'PVS Studio (actions)'
+LICENCE_NAME = 'Free license: ClickHouse, Yandex'
+HTML_REPORT_FOLDER = 'pvs-studio-html-report'
+TXT_REPORT_NAME = 'pvs-studio-task-report.txt'
+
+def process_logs(s3_client, additional_logs, s3_path_prefix):
+    additional_urls = []
+    for log_path in additional_logs:
+        if log_path:
+            additional_urls.append(
+                s3_client.upload_test_report_to_s3(
+                    log_path,
+                    s3_path_prefix + "/" + os.path.basename(log_path)))
+
+    return additional_urls
+
+def _process_txt_report(path):
+    warnings = []
+    errors = []
+    with open(path, 'r') as report_file:
+        for line in report_file:
+            if 'viva64' in line:
+                continue
+            elif 'warn' in line:
+                warnings.append(':'.join(line.split('\t')[0:2]))
+            elif 'err' in line:
+                errors.append(':'.join(line.split('\t')[0:2]))
+    return warnings, errors
+
+def get_commit(gh, commit_sha):
+    repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
+    commit = repo.get_commit(commit_sha)
+    return commit
+
+def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
+    s3_path_prefix = str(pr_number) + "/" + commit_sha + "/" + NAME.lower().replace(' ', '_')
+    additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
+
+    branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
+    branch_name = "master"
+    if pr_number != 0:
+        branch_name = "PR #{}".format(pr_number)
+        branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
+    commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
+
+    task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
+
+    raw_log_url = additional_urls[0]
+    additional_urls.pop(0)
+
+    html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
+    with open('report.html', 'w') as f:
+        f.write(html_report)
+
+    url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
+    logging.info("Search result in url %s", url)
+    return url
+
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    repo_path = os.path.join(os.getenv("REPO_COPY", os.path.abspath("../../")))
+    temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'pvs_check')
+
+    with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
+        event = json.load(event_file)
+    pr_info = PRInfo(event)
+    # this check modify repository so copy it to the temp directory
+    logging.info("Repo copy path %s", repo_path)
+
+    aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
+    aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
+
+    gh = Github(get_best_robot_token())
+
+    images_path = os.path.join(temp_path, 'changed_images.json')
+    docker_image = 'clickhouse/pvs-test'
+    if os.path.exists(images_path):
+        logging.info("Images file exists")
+        with open(images_path, 'r') as images_fd:
+            images = json.load(images_fd)
+            logging.info("Got images %s", images)
+            if 'clickhouse/pvs-test' in images:
+                docker_image += ':' + images['clickhouse/pvs-test']
+
+    logging.info("Got docker image %s", docker_image)
+
+    if not aws_secret_key_id  or not aws_secret_key:
+        logging.info("No secrets, will not upload anything to S3")
+
+    s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
+
+    licence_key = os.getenv('PVS_STUDIO_KEY')
+    cmd = f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --volume={repo_path}:/repo_folder --volume={temp_path}:/test_output -e LICENCE_NAME='{LICENCE_NAME}' -e LICENCE_KEY='{licence_key}' {docker_image}"
+    commit = get_commit(gh, pr_info.sha)
+
+    try:
+        subprocess.check_output(cmd, shell=True)
+    except:
+        commit.create_status(context=NAME, description='PVS report failed to build', state='failure', target_url=f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}")
+        sys.exit(1)
+
+    try:
+        s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_')
+        html_urls = s3_helper.upload_test_folder_to_s3(os.path.join(temp_path, HTML_REPORT_FOLDER), s3_path_prefix)
+        index_html = None
+
+        for url in html_urls:
+            if 'index.html' in url:
+                index_html = '<a href="{}">HTML report</a>'.format(url)
+                break
+
+        if not index_html:
+            commit.create_status(context=NAME, description='PVS report failed to build', state='failure', target_url=f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}")
+            sys.exit(1)
+
+        txt_report = os.path.join(temp_path, TXT_REPORT_NAME)
+        warnings, errors = _process_txt_report(txt_report)
+        errors = errors + warnings
+
+        status = 'success'
+        test_results = [(index_html, "Look at the report"), ("Errors count not checked", "OK")]
+        description = "Total errors {}".format(len(errors))
+        additional_logs = [txt_report, os.path.join(temp_path, 'pvs-studio.log')]
+        report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs)
+
+        print("::notice ::Report url: {}".format(report_url))
+        commit = get_commit(gh, pr_info.sha)
+        commit.create_status(context=NAME, description=description, state=status, target_url=report_url)
+    except Exception as ex:
+        print("Got an exception", ex)
+        sys.exit(1)

tests/ci/report.py

@@ -0,0 +1,298 @@
+# -*- coding: utf-8 -*-
+import os
+import datetime
+
+### FIXME: BEST FRONTEND PRACTICIES BELOW
+
+HTML_BASE_TEST_TEMPLATE = """
+<!DOCTYPE html>
+<html>
+  <style>
+@font-face {{
+    font-family:'Yandex Sans Display Web';
+    src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
+    src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
+            url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
+            url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
+            url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
+            url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
+    font-weight:400;
+    font-style:normal;
+    font-stretch:normal
+}}
+
+body {{ font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }}
+h1 {{ margin-left: 10px; }}
+th, td {{ border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
+td {{ white-space: pre; font-family: Monospace, Courier New; }}
+border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
+a {{ color: #06F; text-decoration: none; }}
+a:hover, a:active {{ color: #F40; text-decoration: underline; }}
+table {{ border: 0; }}
+.main {{ margin-left: 10%; }}
+p.links a {{ padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
+th {{ cursor: pointer; }}
+
+  </style>
+  <title>{title}</title>
+</head>
+<body>
+<div class="main">
+
+<h1>{header}</h1>
+<p class="links">
+<a href="{raw_log_url}">{raw_log_name}</a>
+<a href="{commit_url}">Commit</a>
+{additional_urls}
+<a href="{task_url}">Task (github actions)</a>
+</p>
+{test_part}
+</body>
+<script type="text/javascript">
+    /// Straight from https://stackoverflow.com/questions/14267781/sorting-html-table-with-javascript
+
+    const getCellValue = (tr, idx) => tr.children[idx].innerText || tr.children[idx].textContent;
+
+    const comparer = (idx, asc) => (a, b) => ((v1, v2) =>
+        v1 !== '' && v2 !== '' && !isNaN(v1) && !isNaN(v2) ? v1 - v2 : v1.toString().localeCompare(v2)
+        )(getCellValue(asc ? a : b, idx), getCellValue(asc ? b : a, idx));
+
+    // do the work...
+    document.querySelectorAll('th').forEach(th => th.addEventListener('click', (() => {{
+        const table = th.closest('table');
+        Array.from(table.querySelectorAll('tr:nth-child(n+2)'))
+            .sort(comparer(Array.from(th.parentNode.children).indexOf(th), this.asc = !this.asc))
+            .forEach(tr => table.appendChild(tr) );
+    }})));
+</script>
+</html>
+"""
+
+HTML_TEST_PART = """
+<table>
+<tr>
+{headers}
+</tr>
+{rows}
+</table>
+"""
+
+BASE_HEADERS = ['Test name', 'Test status']
+
+
+def _format_header(header, branch_name, branch_url=None):
+    result = ' '.join([w.capitalize() for w in header.split(' ')])
+    result = result.replace("Clickhouse", "ClickHouse")
+    result = result.replace("clickhouse", "ClickHouse")
+    if 'ClickHouse' not in result:
+        result = 'ClickHouse ' + result
+    result += ' for '
+    if branch_url:
+        result += '<a href="{url}">{name}</a>'.format(url=branch_url, name=branch_name)
+    else:
+        result += branch_name
+    return result
+
+
+def _get_status_style(status):
+    style = "font-weight: bold;"
+    if status in ('OK', 'success', 'PASSED'):
+        style += 'color: #0A0;'
+    elif status in ('FAIL', 'failure', 'error', 'FAILED', 'Timeout'):
+        style += 'color: #F00;'
+    else:
+        style += 'color: #FFB400;'
+    return style
+
+
+def _get_html_url(url):
+    if isinstance(url, str):
+        return '<a href="{url}">{name}</a>'.format(url=url, name=os.path.basename(url))
+    if isinstance(url, tuple):
+        return '<a href="{url}">{name}</a>'.format(url=url[0], name=url[1])
+    return ''
+
+
+def create_test_html_report(header, test_result, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls=[]):
+    if test_result:
+        rows_part = ""
+        num_fails = 0
+        has_test_time = False
+        has_test_logs = False
+        for result in test_result:
+            test_name = result[0]
+            test_status = result[1]
+
+            test_logs = None
+            test_time = None
+            if len(result) > 2:
+                test_time = result[2]
+                has_test_time = True
+
+            if len(result) > 3:
+                test_logs = result[3]
+                has_test_logs = True
+
+            row = "<tr>"
+            row += "<td>" + test_name + "</td>"
+            style = _get_status_style(test_status)
+
+            # Allow to quickly scroll to the first failure.
+            is_fail = test_status == "FAIL" or test_status == 'FLAKY'
+            is_fail_id = ""
+            if is_fail:
+                num_fails = num_fails + 1
+                is_fail_id = 'id="fail' + str(num_fails) + '" '
+
+            row += '<td ' + is_fail_id + 'style="{}">'.format(style) + test_status + "</td>"
+
+            if test_time is not None:
+                row += "<td>" + test_time + "</td>"
+
+            if test_logs is not None:
+                test_logs_html = "<br>".join([_get_html_url(url) for url in test_logs])
+                row += "<td>" + test_logs_html + "</td>"
+
+            row += "</tr>"
+            rows_part += row
+
+        headers = BASE_HEADERS
+        if has_test_time:
+            headers.append('Test time, sec.')
+        if has_test_logs:
+            headers.append('Logs')
+
+        headers = ''.join(['<th>' + h + '</th>' for h in headers])
+        test_part = HTML_TEST_PART.format(headers=headers, rows=rows_part)
+    else:
+        test_part = ""
+
+    additional_html_urls = ""
+    for url in additional_urls:
+        additional_html_urls += ' ' + _get_html_url(url)
+
+    result = HTML_BASE_TEST_TEMPLATE.format(
+        title=_format_header(header, branch_name),
+        header=_format_header(header, branch_name, branch_url),
+        raw_log_name=os.path.basename(raw_log_url),
+        raw_log_url=raw_log_url,
+        task_url=task_url,
+        test_part=test_part,
+        branch_name=branch_name,
+        commit_url=commit_url,
+        additional_urls=additional_html_urls
+    )
+    return result
+
+
+HTML_BASE_BUILD_TEMPLATE = """
+<!DOCTYPE html>
+<html>
+<head>
+  <style>
+@font-face {{
+    font-family:'Yandex Sans Display Web';
+    src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
+    src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
+            url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
+            url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
+            url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
+            url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
+    font-weight:400;
+    font-style:normal;
+    font-stretch:normal
+}}
+
+body {{ font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }}
+h1 {{ margin-left: 10px; }}
+th, td {{ border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
+border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
+a {{ color: #06F; text-decoration: none; }}
+a:hover, a:active {{ color: #F40; text-decoration: underline; }}
+table {{ border: 0; }}
+.main {{ margin: auto; }}
+p.links a {{ padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
+tr:hover td {{filter: brightness(95%);}}
+  </style>
+<title>{title}</title>
+</head>
+<body>
+<div class="main">
+<h1>{header}</h1>
+<table>
+<tr>
+<th>Compiler</th>
+<th>Build type</th>
+<th>Sanitizer</th>
+<th>Bundled</th>
+<th>Splitted</th>
+<th>Status</th>
+<th>Build log</th>
+<th>Build time</th>
+<th class="artifacts">Artifacts</th>
+</tr>
+{rows}
+</table>
+<p class="links">
+<a href="{commit_url}">Commit</a>
+<a href="{task_url}">Task (private network)</a>
+</p>
+</body>
+</html>
+"""
+
+LINK_TEMPLATE = '<a href="{url}">{text}</a>'
+
+
+def create_build_html_report(header, build_results, build_logs_urls, artifact_urls_list, task_url, branch_url, branch_name, commit_url):
+    rows = ""
+    for (build_result, build_log_url, artifact_urls) in zip(build_results, build_logs_urls, artifact_urls_list):
+        row = "<tr>"
+        row += "<td>{}</td>".format(build_result.compiler)
+        if build_result.build_type:
+            row += "<td>{}</td>".format(build_result.build_type)
+        else:
+            row += "<td>{}</td>".format("relwithdebuginfo")
+        if build_result.sanitizer:
+            row += "<td>{}</td>".format(build_result.sanitizer)
+        else:
+            row += "<td>{}</td>".format("none")
+
+        row += "<td>{}</td>".format(build_result.bundled)
+        row += "<td>{}</td>".format(build_result.splitted)
+
+        if build_result.status:
+            style = _get_status_style(build_result.status)
+            row += '<td style="{}">{}</td>'.format(style, build_result.status)
+        else:
+            style = _get_status_style("error")
+            row += '<td style="{}">{}</td>'.format(style, "error")
+
+        row += '<td><a href="{}">link</a></td>'.format(build_log_url)
+
+        if build_result.elapsed_seconds:
+            delta = datetime.timedelta(seconds=build_result.elapsed_seconds)
+        else:
+            delta = 'unknown'
+
+        row += '<td>{}</td>'.format(str(delta))
+
+        links = ""
+        link_separator = "<br/>"
+        if artifact_urls:
+            for artifact_url in artifact_urls:
+                links += LINK_TEMPLATE.format(text=os.path.basename(artifact_url), url=artifact_url)
+                links += link_separator
+            if links:
+                links = links[:-len(link_separator)]
+            row += "<td>{}</td>".format(links)
+
+        row += "</tr>"
+        rows += row
+    return HTML_BASE_BUILD_TEMPLATE.format(
+        title=_format_header(header, branch_name),
+        header=_format_header(header, branch_name, branch_url),
+        rows=rows,
+        task_url=task_url,
+        branch_name=branch_name,
+        commit_url=commit_url)

tests/ci/run_check.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+import os
+import json
+import requests
+from pr_info import PRInfo
+import sys
+import logging
+from github import Github
+from get_robot_token import get_best_robot_token
+
+NAME = 'Run Check (actions)'
+
+TRUSTED_ORG_IDS = {
+    7409213,   # yandex
+    28471076,  # altinity
+    54801242,  # clickhouse
+}
+
+OK_TEST_LABEL = set(["can be tested", "release", "pr-documentation", "pr-doc-fix"])
+DO_NOT_TEST_LABEL = "do not test"
+
+# Individual trusted contirbutors who are not in any trusted organization.
+# Can be changed in runtime: we will append users that we learned to be in
+# a trusted org, to save GitHub API calls.
+TRUSTED_CONTRIBUTORS = {
+    "achimbab",
+    "adevyatova ",  # DOCSUP
+    "Algunenano",   # Raúl Marín, Tinybird
+    "AnaUvarova",   # DOCSUP
+    "anauvarova",   # technical writer, Yandex
+    "annvsh",       # technical writer, Yandex
+    "atereh",       # DOCSUP
+    "azat",
+    "bharatnc",     # Newbie, but already with many contributions.
+    "bobrik",       # Seasoned contributor, CloundFlare
+    "BohuTANG",
+    "damozhaeva",   # DOCSUP
+    "den-crane",
+    "gyuton",       # DOCSUP
+    "gyuton",       # technical writer, Yandex
+    "hagen1778",    # Roman Khavronenko, seasoned contributor
+    "hczhcz",
+    "hexiaoting",   # Seasoned contributor
+    "ildus",        # adjust, ex-pgpro
+    "javisantana",  # a Spanish ClickHouse enthusiast, ex-Carto
+    "ka1bi4",       # DOCSUP
+    "kirillikoff",  # DOCSUP
+    "kitaisreal",   # Seasoned contributor
+    "kreuzerkrieg",
+    "lehasm",       # DOCSUP
+    "michon470",    # DOCSUP
+    "MyroTk",       # Tester in Altinity
+    "myrrc",        # Michael Kot, Altinity
+    "nikvas0",
+    "nvartolomei",
+    "olgarev",      # DOCSUP
+    "otrazhenia",   # Yandex docs contractor
+    "pdv-ru",       # DOCSUP
+    "podshumok",    # cmake expert from QRator Labs
+    "s-mx",         # Maxim Sabyanin, former employee, present contributor
+    "sevirov",      # technical writer, Yandex
+    "spongedu",     # Seasoned contributor
+    "ucasFL",       # Amos Bird's friend
+    "vdimir",       # Employee
+    "vzakaznikov",
+    "YiuRULE",
+    "zlobober"      # Developer of YT
+}
+
+
+def pr_is_by_trusted_user(pr_user_login, pr_user_orgs):
+    if pr_user_login in TRUSTED_CONTRIBUTORS:
+        logging.info("User '{}' is trusted".format(pr_user_login))
+        return True
+
+    logging.info("User '{}' is not trusted".format(pr_user_login))
+
+    for org_id in pr_user_orgs:
+        if org_id in TRUSTED_ORG_IDS:
+            logging.info("Org '{}' is trusted; will mark user {} as trusted".format(org_id, pr_user_login))
+            return True
+        logging.info("Org '{}' is not trusted".format(org_id))
+
+    return False
+
+# Returns whether we should look into individual checks for this PR. If not, it
+# can be skipped entirely.
+def should_run_checks_for_pr(pr_info):
+    # Consider the labels and whether the user is trusted.
+    force_labels = set(['force tests']).intersection(pr_info.labels)
+    if force_labels:
+        return True, "Labeled '{}'".format(', '.join(force_labels))
+
+    if 'do not test' in pr_info.labels:
+        return False, "Labeled 'do not test'"
+
+    if 'can be tested' not in pr_info.labels and not pr_is_by_trusted_user(pr_info.user_login, pr_info.user_orgs):
+        return False, "Needs 'can be tested' label"
+
+    if 'release' in pr_info.labels or 'pr-backport' in pr_info.labels or 'pr-cherrypick' in pr_info.labels:
+        return False, "Don't try new checks for release/backports/cherry-picks"
+
+    return True, "No special conditions apply"
+
+def get_commit(gh, commit_sha):
+    repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
+    commit = repo.get_commit(commit_sha)
+    return commit
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
+        event = json.load(event_file)
+
+    pr_info = PRInfo(event, need_orgs=True)
+    can_run, description = should_run_checks_for_pr(pr_info)
+    gh = Github(get_best_robot_token())
+    commit = get_commit(gh, pr_info.sha)
+    url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
+    if not can_run:
+        print("::notice ::Cannot run")
+        commit.create_status(context=NAME, description=description, state="failure", target_url=url)
+        sys.exit(1)
+    else:
+        print("::notice ::Can run")
+        commit.create_status(context=NAME, description=description, state="pending", target_url=url)

tests/ci/s3_helper.py

@@ -0,0 +1,99 @@
+# -*- coding: utf-8 -*-
+import hashlib
+import logging
+import os
+import boto3
+from botocore.exceptions import ClientError, BotoCoreError
+from multiprocessing.dummy import Pool
+from compress_files import compress_file_fast
+
+def _md5(fname):
+    hash_md5 = hashlib.md5()
+    with open(fname, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hash_md5.update(chunk)
+    logging.debug("MD5 for {} is {}".format(fname, hash_md5.hexdigest()))
+    return hash_md5.hexdigest()
+
+
+def _flatten_list(lst):
+    result = []
+    for elem in lst:
+        if isinstance(elem, list):
+            result += _flatten_list(elem)
+        else:
+            result.append(elem)
+    return result
+
+
+class S3Helper(object):
+    def __init__(self, host, aws_access_key_id, aws_secret_access_key):
+        self.session = boto3.session.Session(aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key)
+        self.client = self.session.client('s3', endpoint_url=host)
+
+    def _upload_file_to_s3(self, bucket_name, file_path, s3_path):
+        logging.debug("Start uploading {} to bucket={} path={}".format(file_path, bucket_name, s3_path))
+        metadata = {}
+        if os.path.getsize(file_path) < 64 * 1024 * 1024:
+            if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"):
+                metadata['ContentType'] = "text/plain; charset=utf-8"
+                logging.info("Content type %s for file path %s", "text/plain; charset=utf-8", file_path)
+            elif s3_path.endswith("html"):
+                metadata['ContentType'] = "text/html; charset=utf-8"
+                logging.info("Content type %s for file path %s", "text/html; charset=utf-8", file_path)
+            else:
+                logging.info("No content type provied for %s", file_path)
+        else:
+            if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"):
+                logging.info("Going to compress file log file %s to %s", file_path, file_path + ".gz")
+                compress_file_fast(file_path, file_path + ".gz")
+                file_path += ".gz"
+                s3_path += ".gz"
+            else:
+                logging.info("Processing file without compression")
+            logging.info("File is too large, do not provide content type")
+
+        self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata)
+        logging.info("Upload {} to {}. Meta: {}".format(file_path, s3_path, metadata))
+        return "https://storage.yandexcloud.net/{bucket}/{path}".format(bucket=bucket_name, path=s3_path)
+
+    def upload_test_report_to_s3(self, file_path, s3_path):
+        return self._upload_file_to_s3('clickhouse-test-reports', file_path, s3_path)
+
+    def upload_build_file_to_s3(self, file_path, s3_path):
+        return self._upload_file_to_s3('clickhouse-builds', file_path, s3_path)
+
+    def _upload_folder_to_s3(self, folder_path, s3_folder_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks):
+        logging.info("Upload folder '{}' to bucket={} of s3 folder '{}'".format(folder_path, bucket_name, s3_folder_path))
+        if not os.path.exists(folder_path):
+            return []
+        files = os.listdir(folder_path)
+        if not files:
+            return []
+
+        p = Pool(min(len(files), 5))
+
+        def task(file_name):
+            full_fs_path = os.path.join(folder_path, file_name)
+            if keep_dirs_in_s3_path:
+                full_s3_path = s3_folder_path + "/" + os.path.basename(folder_path)
+            else:
+                full_s3_path = s3_folder_path
+
+            if os.path.isdir(full_fs_path):
+                return self._upload_folder_to_s3(full_fs_path, full_s3_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks)
+
+            if os.path.islink(full_fs_path):
+                if upload_symlinks:
+                    return self._upload_file_to_s3(bucket_name, full_fs_path, full_s3_path + "/" + file_name)
+                return []
+
+            return self._upload_file_to_s3(bucket_name, full_fs_path, full_s3_path + "/" + file_name)
+
+        return sorted(_flatten_list(list(p.map(task, files))))
+
+    def upload_build_folder_to_s3(self, folder_path, s3_folder_path, keep_dirs_in_s3_path=True, upload_symlinks=True):
+        return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-builds', keep_dirs_in_s3_path, upload_symlinks)
+
+    def upload_test_folder_to_s3(self, folder_path, s3_folder_path):
+        return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-test-reports', True, True)

tests/ci/style_check.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+from github import Github
+from report import create_test_html_report
+import shutil
+import logging
+import subprocess
+import os
+import csv
+from s3_helper import S3Helper
+import time
+import json
+from pr_info import PRInfo
+from get_robot_token import get_best_robot_token
+
+NAME = "Style Check (actions)"
+
+
+def process_logs(s3_client, additional_logs, s3_path_prefix):
+    additional_urls = []
+    for log_path in additional_logs:
+        if log_path:
+            additional_urls.append(
+                s3_client.upload_test_report_to_s3(
+                    log_path,
+                    s3_path_prefix + "/" + os.path.basename(log_path)))
+
+    return additional_urls
+
+
+def process_result(result_folder):
+    test_results = []
+    additional_files = []
+    # Just upload all files from result_folder.
+    # If task provides processed results, then it's responsible for content of result_folder.
+    if os.path.exists(result_folder):
+        test_files = [f for f in os.listdir(result_folder) if os.path.isfile(os.path.join(result_folder, f))]
+        additional_files = [os.path.join(result_folder, f) for f in test_files]
+
+    status_path = os.path.join(result_folder, "check_status.tsv")
+    logging.info("Found test_results.tsv")
+    status = list(csv.reader(open(status_path, 'r'), delimiter='\t'))
+    if len(status) != 1 or len(status[0]) != 2:
+        return "error", "Invalid check_status.tsv", test_results, additional_files
+    state, description = status[0][0], status[0][1]
+
+    try:
+        results_path = os.path.join(result_folder, "test_results.tsv")
+        test_results = list(csv.reader(open(results_path, 'r'), delimiter='\t'))
+        if len(test_results) == 0:
+            raise Exception("Empty results")
+
+        return state, description, test_results, additional_files
+    except Exception:
+        if state == "success":
+            state, description = "error", "Failed to read test_results.tsv"
+        return state, description, test_results, additional_files
+
+def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
+    s3_path_prefix = f"{pr_number}/{commit_sha}/style_check"
+    additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
+
+    branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
+    branch_name = "master"
+    if pr_number != 0:
+        branch_name = "PR #{}".format(pr_number)
+        branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
+    commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
+
+    task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
+
+    raw_log_url = additional_urls[0]
+    additional_urls.pop(0)
+
+    html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
+    with open('report.html', 'w') as f:
+        f.write(html_report)
+
+    url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
+    logging.info("Search result in url %s", url)
+    return url
+
+
+def get_commit(gh, commit_sha):
+    repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
+    commit = repo.get_commit(commit_sha)
+    return commit
+
+def update_check_with_curl(check_id):
+    cmd_template = ("curl -v --request PATCH --url https://api.github.com/repos/ClickHouse/ClickHouse/check-runs/{} "
+           "--header 'authorization: Bearer {}' "
+           "--header 'Accept: application/vnd.github.v3+json' "
+           "--header 'content-type: application/json' "
+           "-d '{{\"name\" : \"hello-world-name\"}}'")
+    cmd = cmd_template.format(check_id, os.getenv("GITHUB_TOKEN"))
+    subprocess.check_call(cmd, shell=True)
+
+if __name__ == "__main__":
+    logging.basicConfig(level=logging.INFO)
+    repo_path = os.path.join(os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../")))
+    temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'style_check')
+
+    with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
+        event = json.load(event_file)
+    pr_info = PRInfo(event)
+
+    if not os.path.exists(temp_path):
+        os.makedirs(temp_path)
+
+    aws_secret_key_id = os.getenv("YANDEX_S3_ACCESS_KEY_ID", "")
+    aws_secret_key = os.getenv("YANDEX_S3_ACCESS_SECRET_KEY", "")
+
+    gh = Github(get_best_robot_token())
+
+    images_path = os.path.join(temp_path, 'changed_images.json')
+    docker_image = 'clickhouse/style-test'
+    if os.path.exists(images_path):
+        logging.info("Images file exists")
+        with open(images_path, 'r') as images_fd:
+            images = json.load(images_fd)
+            logging.info("Got images %s", images)
+            if 'clickhouse/style-test' in images:
+                docker_image += ':' + images['clickhouse/style-test']
+
+    logging.info("Got docker image %s", docker_image)
+    for i in range(10):
+        try:
+            subprocess.check_output(f"docker pull {docker_image}", shell=True)
+            break
+        except Exception as ex:
+            time.sleep(i * 3)
+            logging.info("Got execption pulling docker %s", ex)
+    else:
+        raise Exception(f"Cannot pull dockerhub for image {docker_image}")
+
+    if not aws_secret_key_id  or not aws_secret_key:
+        logging.info("No secrets, will not upload anything to S3")
+
+    s3_helper = S3Helper('https://storage.yandexcloud.net', aws_access_key_id=aws_secret_key_id, aws_secret_access_key=aws_secret_key)
+
+    subprocess.check_output(f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE --volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output {docker_image}", shell=True)
+    state, description, test_results, additional_files = process_result(temp_path)
+    report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_files)
+    print("::notice ::Report url: {}".format(report_url))
+    commit = get_commit(gh, pr_info.sha)
+    commit.create_status(context=NAME, description=description, state=state, target_url=report_url)

tests/ci/termination_lambda/Dockerfile

@@ -0,0 +1,13 @@
+FROM public.ecr.aws/lambda/python:3.9
+
+# Copy function code
+COPY app.py ${LAMBDA_TASK_ROOT}
+
+# Install the function's dependencies using file requirements.txt
+# from your project folder.
+
+COPY requirements.txt  .
+RUN  pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "app.handler" ]

tests/ci/termination_lambda/app.py

@@ -0,0 +1,275 @@
+#!/usr/bin/env python3
+
+import requests
+import argparse
+import jwt
+import sys
+import json
+import time
+from collections import namedtuple
+
+def get_key_and_app_from_aws():
+    import boto3
+    secret_name = "clickhouse_github_secret_key"
+    session = boto3.session.Session()
+    client = session.client(
+        service_name='secretsmanager',
+    )
+    get_secret_value_response = client.get_secret_value(
+        SecretId=secret_name
+    )
+    data = json.loads(get_secret_value_response['SecretString'])
+    return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
+
+def get_installation_id(jwt_token):
+    headers = {
+        "Authorization": f"Bearer {jwt_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+    response = requests.get("https://api.github.com/app/installations", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    return data[0]['id']
+
+def get_access_token(jwt_token, installation_id):
+    headers = {
+        "Authorization": f"Bearer {jwt_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+    response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    return data['token']
+
+
+RunnerDescription = namedtuple('RunnerDescription', ['id', 'name', 'tags', 'offline', 'busy'])
+
+def list_runners(access_token):
+    headers = {
+        "Authorization": f"token {access_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+
+    response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    print("Total runners", data['total_count'])
+    runners = data['runners']
+    result = []
+    for runner in runners:
+        tags = [tag['name'] for tag in runner['labels']]
+        desc = RunnerDescription(id=runner['id'], name=runner['name'], tags=tags,
+                                 offline=runner['status']=='offline', busy=runner['busy'])
+        result.append(desc)
+    return result
+
+def push_metrics_to_cloudwatch(listed_runners, namespace):
+    import boto3
+    client = boto3.client('cloudwatch')
+    metrics_data = []
+    busy_runners = sum(1 for runner in listed_runners if runner.busy)
+    metrics_data.append({
+        'MetricName': 'BusyRunners',
+        'Value': busy_runners,
+        'Unit': 'Count',
+    })
+    total_active_runners = sum(1 for runner in listed_runners if not runner.offline)
+    metrics_data.append({
+        'MetricName': 'ActiveRunners',
+        'Value': total_active_runners,
+        'Unit': 'Count',
+    })
+    total_runners = len(listed_runners)
+    metrics_data.append({
+        'MetricName': 'TotalRunners',
+        'Value': total_runners,
+        'Unit': 'Count',
+    })
+    if total_active_runners == 0:
+        busy_ratio = 100
+    else:
+        busy_ratio = busy_runners / total_active_runners * 100
+
+    metrics_data.append({
+        'MetricName': 'BusyRunnersRatio',
+        'Value': busy_ratio,
+        'Unit': 'Percent',
+    })
+
+    client.put_metric_data(Namespace='RunnersMetrics', MetricData=metrics_data)
+
+
+def how_many_instances_to_kill(event_data):
+    data_array = event_data['CapacityToTerminate']
+    to_kill_by_zone = {}
+    for av_zone in data_array:
+        zone_name = av_zone['AvailabilityZone']
+        to_kill = av_zone['Capacity']
+        if zone_name not in to_kill_by_zone:
+            to_kill_by_zone[zone_name] = 0
+
+        to_kill_by_zone[zone_name] += to_kill
+    return to_kill_by_zone
+
+def get_candidates_to_be_killed(event_data):
+    data_array = event_data['Instances']
+    instances_by_zone = {}
+    for instance in data_array:
+        zone_name = instance['AvailabilityZone']
+        instance_id = instance['InstanceId']
+        if zone_name not in instances_by_zone:
+            instances_by_zone[zone_name] = []
+        instances_by_zone[zone_name].append(instance_id)
+
+    return instances_by_zone
+
+def delete_runner(access_token, runner):
+    headers = {
+        "Authorization": f"token {access_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+
+    response = requests.delete(f"https://api.github.com/orgs/ClickHouse/actions/runners/{runner.id}", headers=headers)
+    response.raise_for_status()
+    print(f"Response code deleting {runner.name} is {response.status_code}")
+    return response.status_code == 204
+
+
+def main(github_secret_key, github_app_id, event):
+    print("Got event", json.dumps(event, sort_keys=True, indent=4))
+    to_kill_by_zone = how_many_instances_to_kill(event)
+    instances_by_zone = get_candidates_to_be_killed(event)
+
+    payload = {
+        "iat": int(time.time()) - 60,
+        "exp": int(time.time()) + (10 * 60),
+        "iss": github_app_id,
+    }
+
+    encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
+    installation_id = get_installation_id(encoded_jwt)
+    access_token = get_access_token(encoded_jwt, installation_id)
+
+    runners = list_runners(access_token)
+
+    to_delete_runners = []
+    instances_to_kill = []
+    for zone in to_kill_by_zone:
+        num_to_kill = to_kill_by_zone[zone]
+        candidates = instances_by_zone[zone]
+        if num_to_kill > len(candidates):
+            raise Exception(f"Required to kill {num_to_kill}, but have only {len(candidates)} candidates in AV {zone}")
+
+        delete_for_av = []
+        for candidate in candidates:
+            if candidate not in set([runner.name for runner in runners]):
+                print(f"Candidate {candidate} was not in runners list, simply delete it")
+                instances_to_kill.append(candidate)
+
+        for candidate in candidates:
+            if len(delete_for_av) + len(instances_to_kill) == num_to_kill:
+                break
+            if candidate in instances_to_kill:
+                continue
+
+            for runner in runners:
+                if runner.name == candidate:
+                    if not runner.busy:
+                        print(f"Runner {runner.name} is not busy and can be deleted from AV {zone}")
+                        delete_for_av.append(runner)
+                    else:
+                        print(f"Runner {runner.name} is busy, not going to delete it")
+                    break
+
+        if len(delete_for_av) < num_to_kill:
+            print(f"Checked all candidates for av {zone}, get to delete {len(delete_for_av)}, but still cannot get required {num_to_kill}")
+        to_delete_runners += delete_for_av
+
+    print("Got instances to kill: ", ', '.join(instances_to_kill))
+    print("Going to delete runners:", ', '.join([runner.name for runner in to_delete_runners]))
+    for runner in to_delete_runners:
+        if delete_runner(access_token, runner):
+            print(f"Runner {runner.name} successfuly deleted from github")
+            instances_to_kill.append(runner.name)
+        else:
+            print(f"Cannot delete {runner.name} from github")
+
+    ## push metrics
+    #runners = list_runners(access_token)
+    #push_metrics_to_cloudwatch(runners, 'RunnersMetrics')
+
+    response = {
+        "InstanceIDs": instances_to_kill
+    }
+    print(response)
+    return response
+
+def handler(event, context):
+    private_key, app_id = get_key_and_app_from_aws()
+    return main(private_key, app_id, event)
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Get list of runners and their states')
+    parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
+    parser.add_argument('-k', '--private-key', help='Private key')
+    parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
+
+    args = parser.parse_args()
+
+    if not args.private_key_path and not args.private_key:
+        print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
+
+    if args.private_key_path and args.private_key:
+        print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
+
+    if args.private_key:
+        private_key = args.private_key
+    else:
+        with open(args.private_key_path, 'r') as key_file:
+            private_key = key_file.read()
+
+    sample_event = {
+        "AutoScalingGroupARN": "arn:aws:autoscaling:us-east-1:<account-id>:autoScalingGroup:d4738357-2d40-4038-ae7e-b00ae0227003:autoScalingGroupName/my-asg",
+        "AutoScalingGroupName": "my-asg",
+        "CapacityToTerminate": [
+            {
+                "AvailabilityZone": "us-east-1b",
+                "Capacity": 1,
+                "InstanceMarketOption": "OnDemand"
+            },
+            {
+                "AvailabilityZone": "us-east-1c",
+                "Capacity": 2,
+                "InstanceMarketOption": "OnDemand"
+            }
+        ],
+        "Instances": [
+            {
+                "AvailabilityZone": "us-east-1b",
+                "InstanceId": "i-08d0b3c1a137e02a5",
+                "InstanceType": "t2.nano",
+                "InstanceMarketOption": "OnDemand"
+            },
+            {
+                "AvailabilityZone": "us-east-1c",
+                "InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal",
+                "InstanceType": "t2.nano",
+                "InstanceMarketOption": "OnDemand"
+            },
+            {
+                "AvailabilityZone": "us-east-1c",
+                "InstanceId": "ip-172-31-27-227.eu-west-1.compute.internal",
+                "InstanceType": "t2.nano",
+                "InstanceMarketOption": "OnDemand"
+            },
+            {
+                "AvailabilityZone": "us-east-1c",
+                "InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal",
+                "InstanceType": "t2.nano",
+                "InstanceMarketOption": "OnDemand"
+            }
+        ],
+        "Cause": "SCALE_IN"
+    }
+
+    main(private_key, args.app_id, sample_event)

tests/ci/termination_lambda/requirements.txt

@@ -0,0 +1,3 @@
+requests
+PyJWT
+cryptography

tests/ci/token_lambda/Dockerfile

@@ -0,0 +1,13 @@
+FROM public.ecr.aws/lambda/python:3.9
+
+# Copy function code
+COPY app.py ${LAMBDA_TASK_ROOT}
+
+# Install the function's dependencies using file requirements.txt
+# from your project folder.
+
+COPY requirements.txt  .
+RUN  pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
+
+# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
+CMD [ "app.handler" ]

tests/ci/token_lambda/app.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+import requests
+import argparse
+import jwt
+import sys
+import json
+import time
+
+def get_installation_id(jwt_token):
+    headers = {
+        "Authorization": f"Bearer {jwt_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+    response = requests.get("https://api.github.com/app/installations", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    return data[0]['id']
+
+def get_access_token(jwt_token, installation_id):
+    headers = {
+        "Authorization": f"Bearer {jwt_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+    response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    return data['token']
+
+def get_runner_registration_token(access_token):
+    headers = {
+        "Authorization": f"token {access_token}",
+        "Accept": "application/vnd.github.v3+json",
+    }
+    response = requests.post("https://api.github.com/orgs/ClickHouse/actions/runners/registration-token", headers=headers)
+    response.raise_for_status()
+    data = response.json()
+    return data['token']
+
+def get_key_and_app_from_aws():
+    import boto3
+    secret_name = "clickhouse_github_secret_key"
+    session = boto3.session.Session()
+    client = session.client(
+        service_name='secretsmanager',
+    )
+    get_secret_value_response = client.get_secret_value(
+        SecretId=secret_name
+    )
+    data = json.loads(get_secret_value_response['SecretString'])
+    return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
+
+
+def main(github_secret_key, github_app_id, push_to_ssm, ssm_parameter_name):
+    payload = {
+        "iat": int(time.time()) - 60,
+        "exp": int(time.time()) + (10 * 60),
+        "iss": github_app_id,
+    }
+
+    encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
+    installation_id = get_installation_id(encoded_jwt)
+    access_token = get_access_token(encoded_jwt, installation_id)
+    runner_registration_token = get_runner_registration_token(access_token)
+
+    if push_to_ssm:
+        import boto3
+
+        print("Trying to put params into ssm manager")
+        client = boto3.client('ssm')
+        client.put_parameter(
+            Name=ssm_parameter_name,
+            Value=runner_registration_token,
+            Type='SecureString',
+            Overwrite=True)
+    else:
+        print("Not push token to AWS Parameter Store, just print:", runner_registration_token)
+
+
+def handler(event, context):
+    private_key, app_id = get_key_and_app_from_aws()
+    main(private_key, app_id, True, 'github_runner_registration_token')
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Get new token from github to add runners')
+    parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
+    parser.add_argument('-k', '--private-key', help='Private key')
+    parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
+    parser.add_argument('--push-to-ssm', action='store_true',  help='Store received token in parameter store')
+    parser.add_argument('--ssm-parameter-name', default='github_runner_registration_token',  help='AWS paramater store parameter name')
+
+    args = parser.parse_args()
+
+    if not args.private_key_path and not args.private_key:
+        print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
+
+    if args.private_key_path and args.private_key:
+        print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
+
+    if args.private_key:
+        private_key = args.private_key
+    else:
+        with open(args.private_key_path, 'r') as key_file:
+            private_key = key_file.read()
+
+    main(private_key, args.app_id, args.push_to_ssm, args.ssm_parameter_name)

tests/ci/token_lambda/requirements.txt

@@ -0,0 +1,3 @@
+requests
+PyJWT
+cryptography

tests/ci/worker/init.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/bash
+set -euo pipefail
+
+echo "Running init script"
+export DEBIAN_FRONTEND=noninteractive
+export RUNNER_HOME=/home/ubuntu/actions-runner
+
+echo "Receiving token"
+export RUNNER_TOKEN=`/usr/local/bin/aws ssm  get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`
+export RUNNER_URL="https://github.com/ClickHouse"
+# Funny fact, but metadata service has fixed IP
+export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`
+
+cd $RUNNER_HOME
+
+echo "Going to configure runner"
+sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64' --work _work
+
+echo "Run"
+sudo -u ubuntu ./run.sh

tests/ci/worker/ubuntu_ami.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+echo "Running prepare script"
+export DEBIAN_FRONTEND=noninteractive
+export RUNNER_VERSION=2.283.1
+export RUNNER_HOME=/home/ubuntu/actions-runner
+
+apt-get update
+
+apt-get install --yes --no-install-recommends \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg \
+    lsb-release \
+    python3-pip \
+    unzip
+
+curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
+
+echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
+
+apt-get update
+
+apt-get install --yes --no-install-recommends docker-ce docker-ce-cli containerd.io
+
+usermod -aG docker ubuntu
+
+pip install boto3 pygithub requests urllib3 unidiff
+
+mkdir -p $RUNNER_HOME && cd $RUNNER_HOME
+
+curl -O -L https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
+
+tar xzf ./actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
+rm -f ./actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
+./bin/installdependencies.sh
+
+chown -R ubuntu:ubuntu $RUNNER_HOME
+
+cd /home/ubuntu
+curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
+unzip awscliv2.zip
+./aws/install
+
+rm -rf /home/ubuntu/awscliv2.zip /home/ubuntu/aws

tests/fuzz/all.dict

@@ -985,10 +985,10 @@
 "RANGE"
 "rank"
 "rankCorr"
-"readWktMultiPolygon"
-"readWktPoint"
-"readWktPolygon"
-"readWktRing"
+"readWKTMultiPolygon"
+"readWKTPoint"
+"readWKTPolygon"
+"readWKTRing"
 "REAL"
 "REFRESH"
 "regexpQuoteMeta"
@@ -1177,6 +1177,7 @@
 "sumWithOverflow"
 "SUSPEND"
 "svg"
+"SVG"
 "SYNC"
 "synonyms"
 "SYNTAX"

tests/fuzz/dictionaries/functions.dict

@@ -52,6 +52,7 @@
 "h3GetResolution"
 "h3EdgeLengthM"
 "svg"
+"SVG"
 "equals"
 "geohashesInBox"
 "polygonsIntersectionCartesian"
@@ -114,7 +115,7 @@
 "replaceOne"
 "emptyArrayInt32"
 "extract"
-"readWktPolygon"
+"readWKTPolygon"
 "notILike"
 "geohashDecode"
 "toModifiedJulianDay"
@@ -164,7 +165,7 @@
 "lessOrEquals"
 "subtractQuarters"
 "ngramSearch"
-"readWktRing"
+"readWKTRing"
 "trimRight"
 "endsWith"
 "ngramDistanceCaseInsensitive"
@@ -713,13 +714,13 @@
 "s2RectContains"
 "toDate"
 "regexpQuoteMeta"
-"readWktMultiPolygon"
+"readWKTMultiPolygon"
 "emptyArrayString"
 "bitmapOr"
 "cutWWW"
 "emptyArrayInt8"
 "less"
-"readWktPoint"
+"readWKTPoint"
 "reinterpretAsDateTime"
 "notEquals"
 "geoToS2"

tests/queries/0_stateless/01300_read_wkt.sql

@@ -1,14 +1,14 @@
-SELECT readWktPoint('POINT(0 0)');
-SELECT readWktPolygon('POLYGON((1 0,10 0,10 10,0 10,1 0))');
-SELECT readWktPolygon('POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))');
-SELECT readWktMultiPolygon('MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))');
+SELECT readWKTPoint('POINT(0 0)');
+SELECT readWKTPolygon('POLYGON((1 0,10 0,10 10,0 10,1 0))');
+SELECT readWKTPolygon('POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))');
+SELECT readWKTMultiPolygon('MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))');
 
 DROP TABLE IF EXISTS geo;
 CREATE TABLE geo (s String, id Int) engine=Memory();
 INSERT INTO geo VALUES ('POINT(0 0)', 1);
 INSERT INTO geo VALUES ('POINT(1 0)', 2);
 INSERT INTO geo VALUES ('POINT(2 0)', 3);
-SELECT readWktPoint(s) FROM geo ORDER BY id;
+SELECT readWKTPoint(s) FROM geo ORDER BY id;
 
 DROP TABLE IF EXISTS geo;
 CREATE TABLE geo (s String, id Int) engine=Memory();
@@ -18,13 +18,13 @@ INSERT INTO geo VALUES ('POLYGON((2 0,10 0,10 10,0 10,2 0))', 3);
 INSERT INTO geo VALUES ('POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))', 4);
 INSERT INTO geo VALUES ('POLYGON((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4))', 5);
 INSERT INTO geo VALUES ('POLYGON((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4))', 6);
-SELECT readWktPolygon(s) FROM geo ORDER BY id;
+SELECT readWKTPolygon(s) FROM geo ORDER BY id;
 
 DROP TABLE IF EXISTS geo;
 CREATE TABLE geo (s String, id Int) engine=Memory();
 INSERT INTO geo VALUES ('MULTIPOLYGON(((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 1);
 INSERT INTO geo VALUES ('MULTIPOLYGON(((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 2);
 INSERT INTO geo VALUES ('MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 3);
-SELECT readWktMultiPolygon(s) FROM geo ORDER BY id;
+SELECT readWKTMultiPolygon(s) FROM geo ORDER BY id;
 
 DROP TABLE geo;

tests/queries/0_stateless/01300_svg.sql

@@ -1,50 +1,50 @@
-SELECT svg((0., 0.));
-SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)]);
-SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]]);
-SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]]);
-SELECT svg((0., 0.), 'b');
-SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)], 'b');
-SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], 'b');
-SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'b');
+SELECT SVG((0., 0.));
+SELECT SVG([(0., 0.), (10, 0), (10, 10), (0, 10)]);
+SELECT SVG([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]]);
+SELECT SVG([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]]);
+SELECT SVG((0., 0.), 'b');
+SELECT SVG([(0., 0.), (10, 0), (10, 10), (0, 10)], 'b');
+SELECT SVG([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], 'b');
+SELECT SVG([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'b');
 
 DROP TABLE IF EXISTS geo;
 CREATE TABLE geo (p Tuple(Float64, Float64), s String, id Int) engine=Memory();
 INSERT INTO geo VALUES ((0., 0.), 'b', 1);
 INSERT INTO geo VALUES ((1., 0.), 'c', 2);
 INSERT INTO geo VALUES ((2., 0.), 'd', 3);
-SELECT svg(p) FROM geo ORDER BY id;
-SELECT svg(p, 'b') FROM geo ORDER BY id;
-SELECT svg((0., 0.), s) FROM geo ORDER BY id;
-SELECT svg(p, s) FROM geo ORDER BY id;
+SELECT SVG(p) FROM geo ORDER BY id;
+SELECT SVG(p, 'b') FROM geo ORDER BY id;
+SELECT SVG((0., 0.), s) FROM geo ORDER BY id;
+SELECT SVG(p, s) FROM geo ORDER BY id;
 
 DROP TABLE IF EXISTS geo;
 CREATE TABLE geo (p Array(Tuple(Float64, Float64)), s String, id Int) engine=Memory();
 INSERT INTO geo VALUES ([(0., 0.), (10, 0), (10, 10), (0, 10)], 'b', 1);
 INSERT INTO geo VALUES ([(1., 0.), (10, 0), (10, 10), (0, 10)], 'c', 2);
 INSERT INTO geo VALUES ([(2., 0.), (10, 0), (10, 10), (0, 10)], 'd', 3);
-SELECT svg(p) FROM geo ORDER BY id;
-SELECT svg(p, 'b') FROM geo ORDER BY id;
-SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)], s) FROM geo ORDER BY id;
-SELECT svg(p, s) FROM geo ORDER BY id;
+SELECT SVG(p) FROM geo ORDER BY id;
+SELECT SVG(p, 'b') FROM geo ORDER BY id;
+SELECT SVG([(0., 0.), (10, 0), (10, 10), (0, 10)], s) FROM geo ORDER BY id;
+SELECT SVG(p, s) FROM geo ORDER BY id;
 
 DROP TABLE IF EXISTS geo;
 CREATE TABLE geo (p Array(Array(Tuple(Float64, Float64))), s String, id Int) engine=Memory();
 INSERT INTO geo VALUES ([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'b', 1);
 INSERT INTO geo VALUES ([[(1., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'c', 2);
 INSERT INTO geo VALUES ([[(2., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'd', 3);
-SELECT svg(p) FROM geo ORDER BY id;
-SELECT svg(p, 'b') FROM geo ORDER BY id;
-SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], s) FROM geo ORDER BY id;
-SELECT svg(p, s) FROM geo ORDER BY id;
+SELECT SVG(p) FROM geo ORDER BY id;
+SELECT SVG(p, 'b') FROM geo ORDER BY id;
+SELECT SVG([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], s) FROM geo ORDER BY id;
+SELECT SVG(p, s) FROM geo ORDER BY id;
 
 DROP TABLE IF EXISTS geo;
 CREATE TABLE geo (p Array(Array(Array(Tuple(Float64, Float64)))), s String, id Int) engine=Memory();
 INSERT INTO geo VALUES ([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'b', 1);
 INSERT INTO geo VALUES ([[[(1., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'c', 2);
 INSERT INTO geo VALUES ([[[(2., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'd', 3);
-SELECT svg(p) FROM geo ORDER BY id;
-SELECT svg(p, 'b') FROM geo ORDER BY id;
-SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], s) FROM geo ORDER BY id;
-SELECT svg(p, s) FROM geo ORDER BY id;
+SELECT SVG(p) FROM geo ORDER BY id;
+SELECT SVG(p, 'b') FROM geo ORDER BY id;
+SELECT SVG([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], s) FROM geo ORDER BY id;
+SELECT SVG(p, s) FROM geo ORDER BY id;
 
 DROP TABLE geo;

tests/queries/0_stateless/02051_symlinks_to_user_files.reference

@@ -0,0 +1 @@
+OK

tests/queries/0_stateless/02051_symlinks_to_user_files.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# Tags: no-fasttest, no-parallel
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+# See 01658_read_file_to_string_column.sh
+user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
+
+FILE_PATH="${user_files_path}/file/"
+mkdir -p ${FILE_PATH}
+chmod 777 ${FILE_PATH}
+
+FILE="test_symlink_${CLICKHOUSE_DATABASE}"
+
+symlink_path=${FILE_PATH}/${FILE}
+file_path=$CUR_DIR/${FILE}
+
+touch ${file_path}
+ln -s ${file_path} ${symlink_path}
+chmod ugo+w ${symlink_path}
+
+function cleanup()
+{
+    rm ${symlink_path} ${file_path}
+}
+trap cleanup EXIT
+
+${CLICKHOUSE_CLIENT} --query="insert into table function file('${symlink_path}', 'Values', 'a String') select 'OK'";
+${CLICKHOUSE_CLIENT} --query="select * from file('${symlink_path}', 'Values', 'a String')";
+

utils/check-style/check-style

@@ -70,7 +70,7 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.xml' |
     xargs xmllint --noout --nonet
 
 # FIXME: for now only clickhouse-test
-pylint --rcfile=$ROOT_PATH/.pylintrc --score=n $ROOT_PATH/tests/clickhouse-test
+pylint --rcfile=$ROOT_PATH/.pylintrc --persistent=no --score=n $ROOT_PATH/tests/clickhouse-test
 
 find $ROOT_PATH -not -path $ROOT_PATH'/contrib*' \( -name '*.yaml' -or -name '*.yml' \) -type f |
     grep -vP $EXCLUDE_DIRS |
@@ -162,7 +162,7 @@ find $ROOT_PATH -name '.gitmodules' | while read i; do grep -F 'url = ' $i | gre
 find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL"
 
 # There shouldn't be any docker containers outside docker directory
-find $ROOT_PATH -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:"
+find $ROOT_PATH -not -path $ROOT_PATH'/tests/ci*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:"
 
 # There shouldn't be any docker compose files outside docker directory
 #find $ROOT_PATH -not -path $ROOT_PATH'/tests/testflows*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name '*compose*.yml' -type f 2>/dev/null | xargs --no-run-if-empty grep -l "version:" | xargs --no-run-if-empty -n1 echo "Please move docker compose to docker directory:"