mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-26 01:22:04 +00:00
Merge branch 'master' into tsv-csv
This commit is contained in:
mergify[bot]
GitHub
commit
0a4360c43e
53
.github/workflows/main.yml
vendored
Normal file
53
.github/workflows/main.yml
vendored
Normal file
@ -0,0 +1,53 @@
|
||||
name: Ligthweight GithubActions
|
||||
on: # yamllint disable-line rule:truthy
|
||||
pull_request:
|
||||
types:
|
||||
- labeled
|
||||
- unlabeled
|
||||
- synchronize
|
||||
- reopened
|
||||
- opened
|
||||
branches:
|
||||
- master
|
||||
jobs:
|
||||
CheckLabels:
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Labels check
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 run_check.py
|
||||
DockerHubPush:
|
||||
needs: CheckLabels
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Images check
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 docker_images_check.py
|
||||
- name: Upload images files to artifacts
|
||||
uses: actions/upload-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/docker_images_check/changed_images.json
|
||||
StyleCheck:
|
||||
needs: DockerHubPush
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Download changed images
|
||||
uses: actions/download-artifact@v2
|
||||
with:
|
||||
name: changed_images
|
||||
path: ${{ runner.temp }}/style_check
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Style Check
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 style_check.py
|
||||
FinishCheck:
|
||||
needs: [StyleCheck, DockerHubPush, CheckLabels]
|
||||
runs-on: [self-hosted]
|
||||
steps:
|
||||
- name: Check out repository code
|
||||
uses: actions/checkout@v2
|
||||
- name: Finish label
|
||||
run: cd $GITHUB_WORKSPACE/tests/ci && python3 finish_check.py
|
||||
2
.gitmodules
vendored
2
.gitmodules
vendored
@ -140,7 +140,7 @@
|
||||
url = https://github.com/ClickHouse-Extras/libc-headers.git
|
||||
[submodule "contrib/replxx"]
|
||||
path = contrib/replxx
|
||||
url = https://github.com/ClickHouse-Extras/replxx.git
|
||||
url = https://github.com/AmokHuginnsson/replxx.git
|
||||
[submodule "contrib/avro"]
|
||||
path = contrib/avro
|
||||
url = https://github.com/ClickHouse-Extras/avro.git
|
||||
|
||||
@ -177,6 +177,10 @@ ReplxxLineReader::ReplxxLineReader(
|
||||
/// bind C-p/C-n to history-previous/history-next like readline.
|
||||
rx.bind_key(Replxx::KEY::control('N'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_NEXT, code); });
|
||||
rx.bind_key(Replxx::KEY::control('P'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::HISTORY_PREVIOUS, code); });
|
||||
|
||||
/// bind C-j to ENTER action.
|
||||
rx.bind_key(Replxx::KEY::control('J'), [this](char32_t code) { return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); });
|
||||
|
||||
/// By default COMPLETE_NEXT/COMPLETE_PREV was binded to C-p/C-n, re-bind
|
||||
/// to M-P/M-N (that was used for HISTORY_COMMON_PREFIX_SEARCH before, but
|
||||
/// it also binded to M-p/M-n).
|
||||
|
||||
@ -34,8 +34,6 @@ endif()
|
||||
if (CAPNP_LIBRARIES)
|
||||
set (USE_CAPNP 1)
|
||||
elseif(NOT MISSING_INTERNAL_CAPNP_LIBRARY)
|
||||
add_subdirectory(contrib/capnproto-cmake)
|
||||
|
||||
set (CAPNP_LIBRARIES capnpc)
|
||||
set (USE_CAPNP 1)
|
||||
set (USE_INTERNAL_CAPNP_LIBRARY 1)
|
||||
|
||||
88
contrib/CMakeLists.txt
vendored
88
contrib/CMakeLists.txt
vendored
@ -1,16 +1,5 @@
|
||||
# Third-party libraries may have substandard code.
|
||||
|
||||
# Put all targets defined here and in added subfolders under "contrib/" folder in GUI-based IDEs by default.
|
||||
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they will
|
||||
# appear not in "contrib/" as originally planned here.
|
||||
get_filename_component (_current_dir_name "${CMAKE_CURRENT_LIST_DIR}" NAME)
|
||||
if (CMAKE_FOLDER)
|
||||
set (CMAKE_FOLDER "${CMAKE_FOLDER}/${_current_dir_name}")
|
||||
else ()
|
||||
set (CMAKE_FOLDER "${_current_dir_name}")
|
||||
endif ()
|
||||
unset (_current_dir_name)
|
||||
|
||||
set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
|
||||
set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
|
||||
|
||||
@ -49,6 +38,10 @@ add_subdirectory (replxx-cmake)
|
||||
add_subdirectory (unixodbc-cmake)
|
||||
add_subdirectory (nanodbc-cmake)
|
||||
|
||||
if (USE_INTERNAL_CAPNP_LIBRARY AND NOT MISSING_INTERNAL_CAPNP_LIBRARY)
|
||||
add_subdirectory(capnproto-cmake)
|
||||
endif ()
|
||||
|
||||
if (ENABLE_FUZZING)
|
||||
add_subdirectory (libprotobuf-mutator-cmake)
|
||||
endif()
|
||||
@ -352,3 +345,76 @@ endif()
|
||||
if (USE_S2_GEOMETRY)
|
||||
add_subdirectory(s2geometry-cmake)
|
||||
endif()
|
||||
|
||||
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
|
||||
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
|
||||
# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,
|
||||
# instead of controlling it via CMAKE_FOLDER.
|
||||
|
||||
function (ensure_target_rooted_in _target _folder)
|
||||
# Skip INTERFACE library targets, since FOLDER property is not available for them.
|
||||
get_target_property (_target_type "${_target}" TYPE)
|
||||
if (_target_type STREQUAL "INTERFACE_LIBRARY")
|
||||
return ()
|
||||
endif ()
|
||||
|
||||
# Read the original FOLDER property value, if any.
|
||||
get_target_property (_folder_prop "${_target}" FOLDER)
|
||||
|
||||
# Normalize that value, so we avoid possible repetitions in folder names.
|
||||
|
||||
if (NOT _folder_prop)
|
||||
set (_folder_prop "")
|
||||
endif ()
|
||||
|
||||
if (CMAKE_FOLDER AND _folder_prop MATCHES "^${CMAKE_FOLDER}/(.*)\$")
|
||||
set (_folder_prop "${CMAKE_MATCH_1}")
|
||||
endif ()
|
||||
|
||||
if (_folder AND _folder_prop MATCHES "^${_folder}/(.*)\$")
|
||||
set (_folder_prop "${CMAKE_MATCH_1}")
|
||||
endif ()
|
||||
|
||||
if (_folder)
|
||||
set (_folder_prop "${_folder}/${_folder_prop}")
|
||||
endif ()
|
||||
|
||||
if (CMAKE_FOLDER)
|
||||
set (_folder_prop "${CMAKE_FOLDER}/${_folder_prop}")
|
||||
endif ()
|
||||
|
||||
# Set the updated FOLDER property value back.
|
||||
set_target_properties ("${_target}" PROPERTIES FOLDER "${_folder_prop}")
|
||||
endfunction ()
|
||||
|
||||
function (ensure_own_targets_are_rooted_in _dir _folder)
|
||||
get_directory_property (_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS)
|
||||
foreach (_target IN LISTS _targets)
|
||||
ensure_target_rooted_in ("${_target}" "${_folder}")
|
||||
endforeach ()
|
||||
endfunction ()
|
||||
|
||||
function (ensure_all_targets_are_rooted_in _dir _folder)
|
||||
ensure_own_targets_are_rooted_in ("${_dir}" "${_folder}")
|
||||
|
||||
get_property (_sub_dirs DIRECTORY "${_dir}" PROPERTY SUBDIRECTORIES)
|
||||
foreach (_sub_dir IN LISTS _sub_dirs)
|
||||
ensure_all_targets_are_rooted_in ("${_sub_dir}" "${_folder}")
|
||||
endforeach ()
|
||||
endfunction ()
|
||||
|
||||
function (organize_ide_folders_2_level _dir)
|
||||
get_filename_component (_dir_name "${_dir}" NAME)
|
||||
ensure_own_targets_are_rooted_in ("${_dir}" "${_dir_name}")
|
||||
|
||||
# Note, that we respect only first two levels of nesting, we don't want to
|
||||
# reorganize target folders further within each third-party dir.
|
||||
|
||||
get_property (_sub_dirs DIRECTORY "${_dir}" PROPERTY SUBDIRECTORIES)
|
||||
foreach (_sub_dir IN LISTS _sub_dirs)
|
||||
get_filename_component (_sub_dir_name "${_sub_dir}" NAME)
|
||||
ensure_all_targets_are_rooted_in ("${_sub_dir}" "${_dir_name}/${_sub_dir_name}")
|
||||
endforeach ()
|
||||
endfunction ()
|
||||
|
||||
organize_ide_folders_2_level ("${CMAKE_CURRENT_LIST_DIR}")
|
||||
|
||||
@ -47,6 +47,7 @@ set(SRCS
|
||||
)
|
||||
|
||||
add_library(cxx ${SRCS})
|
||||
set_target_properties(cxx PROPERTIES FOLDER "contrib/libcxx-cmake")
|
||||
|
||||
target_include_directories(cxx SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBCXX_SOURCE_DIR}/include>)
|
||||
target_compile_definitions(cxx PRIVATE -D_LIBCPP_BUILDING_LIBRARY -DLIBCXX_BUILDING_LIBCXXABI)
|
||||
|
||||
@ -22,6 +22,7 @@ set(SRCS
|
||||
)
|
||||
|
||||
add_library(cxxabi ${SRCS})
|
||||
set_target_properties(cxxabi PROPERTIES FOLDER "contrib/libcxxabi-cmake")
|
||||
|
||||
# Third party library may have substandard code.
|
||||
target_compile_options(cxxabi PRIVATE -w)
|
||||
|
||||
@ -39,6 +39,7 @@ set(LIBUNWIND_SOURCES
|
||||
${LIBUNWIND_ASM_SOURCES})
|
||||
|
||||
add_library(unwind ${LIBUNWIND_SOURCES})
|
||||
set_target_properties(unwind PROPERTIES FOLDER "contrib/libunwind-cmake")
|
||||
|
||||
target_include_directories(unwind SYSTEM BEFORE PUBLIC $<BUILD_INTERFACE:${LIBUNWIND_SOURCE_DIR}/include>)
|
||||
target_compile_definitions(unwind PRIVATE -D_LIBUNWIND_NO_HEAP=1 -D_DEBUG -D_LIBUNWIND_IS_NATIVE_ONLY)
|
||||
|
||||
2
contrib/replxx
vendored
2
contrib/replxx
vendored
@ -1 +1 @@
|
||||
Subproject commit f97765df14f4a6236d69b8f14b53ef2051ebd95a
|
||||
Subproject commit b0c266c2d8a835784181e17292b421848c78c6b8
|
||||
@ -1,16 +1,22 @@
|
||||
# docker build -t clickhouse/kerberized-hadoop .
|
||||
|
||||
FROM sequenceiq/hadoop-docker:2.7.0
|
||||
RUN sed -i -e 's/^\#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo
|
||||
RUN sed -i -e 's/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Base.repo
|
||||
RUN sed -i -e 's#http://mirror.centos.org/#http://vault.centos.org/#' /etc/yum.repos.d/CentOS-Base.repo
|
||||
|
||||
RUN sed -i -e 's/^\#baseurl/baseurl/' /etc/yum.repos.d/CentOS-Base.repo && \
|
||||
sed -i -e 's/^mirrorlist/#mirrorlist/' /etc/yum.repos.d/CentOS-Base.repo && \
|
||||
sed -i -e 's#http://mirror.centos.org/#http://vault.centos.org/#' /etc/yum.repos.d/CentOS-Base.repo
|
||||
|
||||
# https://community.letsencrypt.org/t/rhel-centos-6-openssl-client-compatibility-after-dst-root-ca-x3-expiration/161032/81
|
||||
RUN sed -i s/xMDkzMDE0MDExNVow/0MDkzMDE4MTQwM1ow/ /etc/pki/tls/certs/ca-bundle.crt
|
||||
|
||||
RUN yum clean all && \
|
||||
rpm --rebuilddb && \
|
||||
yum -y update && \
|
||||
yum -y install yum-plugin-ovl && \
|
||||
yum --quiet -y install krb5-workstation.x86_64
|
||||
|
||||
RUN cd /tmp && \
|
||||
curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \
|
||||
curl http://archive.apache.org/dist/commons/daemon/source/commons-daemon-1.0.15-src.tar.gz -o commons-daemon-1.0.15-src.tar.gz && \
|
||||
tar xzf commons-daemon-1.0.15-src.tar.gz && \
|
||||
cd commons-daemon-1.0.15-src/src/native/unix && \
|
||||
./configure && \
|
||||
|
||||
@ -37,7 +37,9 @@ RUN set -x \
|
||||
|| echo "WARNING: Some file was just downloaded from the internet without any validation and we are installing it into the system"; } \
|
||||
&& dpkg -i "${PKG_VERSION}.deb"
|
||||
|
||||
CMD echo "Running PVS version $PKG_VERSION" && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
|
||||
ENV CCACHE_DIR=/test_output/ccache
|
||||
|
||||
CMD echo "Running PVS version $PKG_VERSION" && mkdir -p $CCACHE_DIR && cd /repo_folder && pvs-studio-analyzer credentials $LICENCE_NAME $LICENCE_KEY -o ./licence.lic \
|
||||
&& cmake . -D"ENABLE_EMBEDDED_COMPILER"=OFF -D"USE_INTERNAL_PROTOBUF_LIBRARY"=OFF -D"USE_INTERNAL_GRPC_LIBRARY"=OFF -DCMAKE_C_COMPILER=clang-13 -DCMAKE_CXX_COMPILER=clang\+\+-13 \
|
||||
&& ninja re2_st clickhouse_grpc_protos \
|
||||
&& pvs-studio-analyzer analyze -o pvs-studio.log -e contrib -j 4 -l ./licence.lic; \
|
||||
|
||||
@ -1,5 +1,7 @@
|
||||
#!/bin/bash
|
||||
|
||||
# yaml check is not the best one
|
||||
|
||||
cd /ClickHouse/utils/check-style || echo -e "failure\tRepo not found" > /test_output/check_status.tsv
|
||||
./check-style -n |& tee /test_output/style_output.txt
|
||||
./check-typos |& tee /test_output/typos_output.txt
|
||||
|
||||
@ -128,6 +128,8 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
|
||||
- `--history_file` — Path to a file containing command history.
|
||||
- `--param_<name>` — Value for a [query with parameters](#cli-queries-with-parameters).
|
||||
- `--hardware-utilization` — Print hardware utilization information in progress bar.
|
||||
- `--print-profile-events` – Print `ProfileEvents` packets.
|
||||
- `--profile-events-delay-ms` – Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet).
|
||||
|
||||
Since version 20.5, `clickhouse-client` has automatic syntax highlighting (always enabled).
|
||||
|
||||
|
||||
@ -502,19 +502,16 @@ void LocalServer::processConfig()
|
||||
|
||||
format = config().getString("output-format", config().getString("format", is_interactive ? "PrettyCompact" : "TSV"));
|
||||
insert_format = "Values";
|
||||
|
||||
/// Setting value from cmd arg overrides one from config
|
||||
if (global_context->getSettingsRef().max_insert_block_size.changed)
|
||||
insert_format_max_block_size = global_context->getSettingsRef().max_insert_block_size;
|
||||
else
|
||||
insert_format_max_block_size = config().getInt("insert_format_max_block_size", global_context->getSettingsRef().max_insert_block_size);
|
||||
|
||||
/// Skip networking
|
||||
|
||||
/// Sets external authenticators config (LDAP, Kerberos).
|
||||
global_context->setExternalAuthenticatorsConfig(config());
|
||||
|
||||
global_context->initializeBackgroundExecutors();
|
||||
|
||||
setupUsers();
|
||||
|
||||
/// Limit on total number of concurrently executing queries.
|
||||
|
||||
@ -919,7 +919,7 @@ if (ThreadFuzzer::instance().isEffective())
|
||||
|
||||
/// Initialize background executors after we load default_profile config.
|
||||
/// This is needed to load proper values of background_pool_size etc.
|
||||
global_context->initializeBackgroundExecutors();
|
||||
global_context->initializeBackgroundExecutorsIfNeeded();
|
||||
|
||||
if (settings.async_insert_threads)
|
||||
global_context->setAsynchronousInsertQueue(std::make_shared<AsynchronousInsertQueue>(
|
||||
|
||||
@ -267,7 +267,7 @@ void ClientBase::onLogData(Block & block)
|
||||
{
|
||||
initLogsOutputStream();
|
||||
progress_indication.clearProgressOutput();
|
||||
logs_out_stream->write(block);
|
||||
logs_out_stream->writeLogs(block);
|
||||
logs_out_stream->flush();
|
||||
}
|
||||
|
||||
@ -669,39 +669,61 @@ void ClientBase::onEndOfStream()
|
||||
void ClientBase::onProfileEvents(Block & block)
|
||||
{
|
||||
const auto rows = block.rows();
|
||||
if (rows == 0 || !progress_indication.print_hardware_utilization)
|
||||
if (rows == 0)
|
||||
return;
|
||||
const auto & array_thread_id = typeid_cast<const ColumnUInt64 &>(*block.getByName("thread_id").column).getData();
|
||||
const auto & names = typeid_cast<const ColumnString &>(*block.getByName("name").column);
|
||||
const auto & host_names = typeid_cast<const ColumnString &>(*block.getByName("host_name").column);
|
||||
const auto & array_values = typeid_cast<const ColumnUInt64 &>(*block.getByName("value").column).getData();
|
||||
|
||||
const auto * user_time_name = ProfileEvents::getName(ProfileEvents::UserTimeMicroseconds);
|
||||
const auto * system_time_name = ProfileEvents::getName(ProfileEvents::SystemTimeMicroseconds);
|
||||
|
||||
HostToThreadTimesMap thread_times;
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
if (progress_indication.print_hardware_utilization)
|
||||
{
|
||||
auto thread_id = array_thread_id[i];
|
||||
auto host_name = host_names.getDataAt(i).toString();
|
||||
if (thread_id != 0)
|
||||
progress_indication.addThreadIdToList(host_name, thread_id);
|
||||
auto event_name = names.getDataAt(i);
|
||||
auto value = array_values[i];
|
||||
if (event_name == user_time_name)
|
||||
const auto & array_thread_id = typeid_cast<const ColumnUInt64 &>(*block.getByName("thread_id").column).getData();
|
||||
const auto & names = typeid_cast<const ColumnString &>(*block.getByName("name").column);
|
||||
const auto & host_names = typeid_cast<const ColumnString &>(*block.getByName("host_name").column);
|
||||
const auto & array_values = typeid_cast<const ColumnUInt64 &>(*block.getByName("value").column).getData();
|
||||
|
||||
const auto * user_time_name = ProfileEvents::getName(ProfileEvents::UserTimeMicroseconds);
|
||||
const auto * system_time_name = ProfileEvents::getName(ProfileEvents::SystemTimeMicroseconds);
|
||||
|
||||
HostToThreadTimesMap thread_times;
|
||||
for (size_t i = 0; i < rows; ++i)
|
||||
{
|
||||
thread_times[host_name][thread_id].user_ms = value;
|
||||
auto thread_id = array_thread_id[i];
|
||||
auto host_name = host_names.getDataAt(i).toString();
|
||||
if (thread_id != 0)
|
||||
progress_indication.addThreadIdToList(host_name, thread_id);
|
||||
auto event_name = names.getDataAt(i);
|
||||
auto value = array_values[i];
|
||||
if (event_name == user_time_name)
|
||||
{
|
||||
thread_times[host_name][thread_id].user_ms = value;
|
||||
}
|
||||
else if (event_name == system_time_name)
|
||||
{
|
||||
thread_times[host_name][thread_id].system_ms = value;
|
||||
}
|
||||
else if (event_name == MemoryTracker::USAGE_EVENT_NAME)
|
||||
{
|
||||
thread_times[host_name][thread_id].memory_usage = value;
|
||||
}
|
||||
}
|
||||
else if (event_name == system_time_name)
|
||||
progress_indication.updateThreadEventData(thread_times);
|
||||
}
|
||||
|
||||
if (profile_events.print)
|
||||
{
|
||||
if (profile_events.watch.elapsedMilliseconds() >= profile_events.delay_ms)
|
||||
{
|
||||
thread_times[host_name][thread_id].system_ms = value;
|
||||
initLogsOutputStream();
|
||||
progress_indication.clearProgressOutput();
|
||||
logs_out_stream->writeProfileEvents(block);
|
||||
logs_out_stream->flush();
|
||||
|
||||
profile_events.watch.restart();
|
||||
profile_events.last_block = {};
|
||||
}
|
||||
else if (event_name == MemoryTracker::USAGE_EVENT_NAME)
|
||||
else
|
||||
{
|
||||
thread_times[host_name][thread_id].memory_usage = value;
|
||||
profile_events.last_block = block;
|
||||
}
|
||||
}
|
||||
progress_indication.updateThreadEventData(thread_times);
|
||||
}
|
||||
|
||||
|
||||
@ -1024,6 +1046,7 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
|
||||
processed_rows = 0;
|
||||
written_first_block = false;
|
||||
progress_indication.resetProgress();
|
||||
profile_events.watch.restart();
|
||||
|
||||
{
|
||||
/// Temporarily apply query settings to context.
|
||||
@ -1092,6 +1115,15 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
|
||||
}
|
||||
}
|
||||
|
||||
/// Always print last block (if it was not printed already)
|
||||
if (profile_events.last_block)
|
||||
{
|
||||
initLogsOutputStream();
|
||||
progress_indication.clearProgressOutput();
|
||||
logs_out_stream->writeProfileEvents(profile_events.last_block);
|
||||
logs_out_stream->flush();
|
||||
}
|
||||
|
||||
if (is_interactive)
|
||||
{
|
||||
std::cout << std::endl << processed_rows << " rows in set. Elapsed: " << progress_indication.elapsedSeconds() << " sec. ";
|
||||
@ -1582,6 +1614,8 @@ void ClientBase::init(int argc, char ** argv)
|
||||
("ignore-error", "do not stop processing in multiquery mode")
|
||||
("stacktrace", "print stack traces of exceptions")
|
||||
("hardware-utilization", "print hardware utilization information in progress bar")
|
||||
("print-profile-events", po::value(&profile_events.print)->zero_tokens(), "Printing ProfileEvents packets")
|
||||
("profile-events-delay-ms", po::value<UInt64>()->default_value(profile_events.delay_ms), "Delay between printing `ProfileEvents` packets (-1 - print only totals, 0 - print every single packet)")
|
||||
;
|
||||
|
||||
addOptions(options_description);
|
||||
@ -1633,6 +1667,10 @@ void ClientBase::init(int argc, char ** argv)
|
||||
config().setBool("vertical", true);
|
||||
if (options.count("stacktrace"))
|
||||
config().setBool("stacktrace", true);
|
||||
if (options.count("print-profile-events"))
|
||||
config().setBool("print-profile-events", true);
|
||||
if (options.count("profile-events-delay-ms"))
|
||||
config().setInt("profile-events-delay-ms", options["profile-events-delay-ms"].as<UInt64>());
|
||||
if (options.count("progress"))
|
||||
config().setBool("progress", true);
|
||||
if (options.count("echo"))
|
||||
@ -1653,6 +1691,8 @@ void ClientBase::init(int argc, char ** argv)
|
||||
progress_indication.print_hardware_utilization = true;
|
||||
|
||||
query_processing_stage = QueryProcessingStage::fromString(options["stage"].as<std::string>());
|
||||
profile_events.print = options.count("print-profile-events");
|
||||
profile_events.delay_ms = options["profile-events-delay-ms"].as<UInt64>();
|
||||
|
||||
processOptions(options_description, options, external_tables_arguments);
|
||||
argsToConfig(common_arguments, config(), 100);
|
||||
|
||||
@ -3,6 +3,7 @@
|
||||
#include <Common/ProgressIndication.h>
|
||||
#include <Common/InterruptListener.h>
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
#include <Core/ExternalTable.h>
|
||||
#include <Poco/Util/Application.h>
|
||||
#include <Interpreters/Context.h>
|
||||
@ -218,6 +219,16 @@ protected:
|
||||
QueryFuzzer fuzzer;
|
||||
int query_fuzzer_runs = 0;
|
||||
|
||||
struct
|
||||
{
|
||||
bool print = false;
|
||||
/// UINT64_MAX -- print only last
|
||||
UInt64 delay_ms = 0;
|
||||
Stopwatch watch;
|
||||
/// For printing only last (delay_ms == 0).
|
||||
Block last_block;
|
||||
} profile_events;
|
||||
|
||||
QueryProcessingStage::Enum query_processing_stage;
|
||||
};
|
||||
|
||||
|
||||
@ -109,29 +109,29 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
|
||||
{TokenType::OpeningSquareBracket, Replxx::Color::BROWN},
|
||||
{TokenType::ClosingSquareBracket, Replxx::Color::BROWN},
|
||||
{TokenType::DoubleColon, Replxx::Color::BROWN},
|
||||
{TokenType::OpeningCurlyBrace, Replxx::Color::INTENSE},
|
||||
{TokenType::ClosingCurlyBrace, Replxx::Color::INTENSE},
|
||||
{TokenType::OpeningCurlyBrace, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::ClosingCurlyBrace, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
|
||||
{TokenType::Comma, Replxx::Color::INTENSE},
|
||||
{TokenType::Semicolon, Replxx::Color::INTENSE},
|
||||
{TokenType::Dot, Replxx::Color::INTENSE},
|
||||
{TokenType::Asterisk, Replxx::Color::INTENSE},
|
||||
{TokenType::Comma, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Semicolon, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Dot, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Asterisk, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::HereDoc, Replxx::Color::CYAN},
|
||||
{TokenType::Plus, Replxx::Color::INTENSE},
|
||||
{TokenType::Minus, Replxx::Color::INTENSE},
|
||||
{TokenType::Slash, Replxx::Color::INTENSE},
|
||||
{TokenType::Percent, Replxx::Color::INTENSE},
|
||||
{TokenType::Arrow, Replxx::Color::INTENSE},
|
||||
{TokenType::QuestionMark, Replxx::Color::INTENSE},
|
||||
{TokenType::Colon, Replxx::Color::INTENSE},
|
||||
{TokenType::Equals, Replxx::Color::INTENSE},
|
||||
{TokenType::NotEquals, Replxx::Color::INTENSE},
|
||||
{TokenType::Less, Replxx::Color::INTENSE},
|
||||
{TokenType::Greater, Replxx::Color::INTENSE},
|
||||
{TokenType::LessOrEquals, Replxx::Color::INTENSE},
|
||||
{TokenType::GreaterOrEquals, Replxx::Color::INTENSE},
|
||||
{TokenType::Concatenation, Replxx::Color::INTENSE},
|
||||
{TokenType::At, Replxx::Color::INTENSE},
|
||||
{TokenType::Plus, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Minus, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Slash, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Percent, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Arrow, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::QuestionMark, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Colon, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Equals, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::NotEquals, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Less, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Greater, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::LessOrEquals, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::GreaterOrEquals, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::Concatenation, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::At, replxx::color::bold(Replxx::Color::DEFAULT)},
|
||||
{TokenType::DoubleAt, Replxx::Color::MAGENTA},
|
||||
|
||||
{TokenType::EndOfStream, Replxx::Color::DEFAULT},
|
||||
@ -142,7 +142,7 @@ void highlight(const String & query, std::vector<replxx::Replxx::Color> & colors
|
||||
{TokenType::ErrorDoubleQuoteIsNotClosed, Replxx::Color::RED},
|
||||
{TokenType::ErrorSinglePipeMark, Replxx::Color::RED},
|
||||
{TokenType::ErrorWrongNumber, Replxx::Color::RED},
|
||||
{ TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED }};
|
||||
{TokenType::ErrorMaxQuerySizeExceeded, Replxx::Color::RED}};
|
||||
|
||||
const Replxx::Color unknown_token_color = Replxx::Color::RED;
|
||||
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
#include <Client/InternalTextLogs.h>
|
||||
#include <Core/Block.h>
|
||||
#include <Interpreters/InternalTextLogsQueue.h>
|
||||
#include <Interpreters/ProfileEventsExt.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/HashTable/Hash.h>
|
||||
#include <DataTypes/IDataType.h>
|
||||
@ -13,7 +14,7 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
void InternalTextLogs::write(const Block & block)
|
||||
void InternalTextLogs::writeLogs(const Block & block)
|
||||
{
|
||||
const auto & array_event_time = typeid_cast<const ColumnUInt32 &>(*block.getByName("event_time").column).getData();
|
||||
const auto & array_microseconds = typeid_cast<const ColumnUInt32 &>(*block.getByName("event_time_microseconds").column).getData();
|
||||
@ -97,4 +98,69 @@ void InternalTextLogs::write(const Block & block)
|
||||
}
|
||||
}
|
||||
|
||||
void InternalTextLogs::writeProfileEvents(const Block & block)
|
||||
{
|
||||
const auto & column_host_name = typeid_cast<const ColumnString &>(*block.getByName("host_name").column);
|
||||
const auto & array_current_time = typeid_cast<const ColumnUInt32 &>(*block.getByName("current_time").column).getData();
|
||||
const auto & array_thread_id = typeid_cast<const ColumnUInt64 &>(*block.getByName("thread_id").column).getData();
|
||||
const auto & array_type = typeid_cast<const ColumnInt8 &>(*block.getByName("type").column).getData();
|
||||
const auto & column_name = typeid_cast<const ColumnString &>(*block.getByName("name").column);
|
||||
const auto & array_value = typeid_cast<const ColumnUInt64 &>(*block.getByName("value").column).getData();
|
||||
|
||||
for (size_t row_num = 0; row_num < block.rows(); ++row_num)
|
||||
{
|
||||
/// host_name
|
||||
auto host_name = column_host_name.getDataAt(row_num);
|
||||
if (host_name.size)
|
||||
{
|
||||
writeCString("[", wb);
|
||||
if (color)
|
||||
writeString(setColor(StringRefHash()(host_name)), wb);
|
||||
writeString(host_name, wb);
|
||||
if (color)
|
||||
writeCString(resetColor(), wb);
|
||||
writeCString("] ", wb);
|
||||
}
|
||||
|
||||
/// current_time
|
||||
auto current_time = array_current_time[row_num];
|
||||
writeDateTimeText<'.', ':'>(current_time, wb);
|
||||
|
||||
/// thread_id
|
||||
UInt64 thread_id = array_thread_id[row_num];
|
||||
writeCString(" [ ", wb);
|
||||
if (color)
|
||||
writeString(setColor(intHash64(thread_id)), wb);
|
||||
writeIntText(thread_id, wb);
|
||||
if (color)
|
||||
writeCString(resetColor(), wb);
|
||||
writeCString(" ] ", wb);
|
||||
|
||||
/// name
|
||||
auto name = column_name.getDataAt(row_num);
|
||||
if (color)
|
||||
writeString(setColor(StringRefHash()(name)), wb);
|
||||
DB::writeString(name, wb);
|
||||
if (color)
|
||||
writeCString(resetColor(), wb);
|
||||
writeCString(": ", wb);
|
||||
|
||||
/// value
|
||||
UInt64 value = array_value[row_num];
|
||||
writeIntText(value, wb);
|
||||
|
||||
//// type
|
||||
Int8 type = array_type[row_num];
|
||||
writeCString(" (", wb);
|
||||
if (color)
|
||||
writeString(setColor(intHash64(type)), wb);
|
||||
writeString(toString(ProfileEvents::TypeEnum->castToName(type)), wb);
|
||||
if (color)
|
||||
writeCString(resetColor(), wb);
|
||||
writeCString(")", wb);
|
||||
|
||||
writeChar('\n', wb);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -6,16 +6,37 @@
|
||||
namespace DB
|
||||
{
|
||||
|
||||
/// Prints internal server logs
|
||||
/// Input blocks have to have the same structure as SystemLogsQueue::getSampleBlock()
|
||||
/// Prints internal server logs or profile events with colored output (if requested).
|
||||
/// NOTE: IRowOutputFormat does not suite well for this case
|
||||
class InternalTextLogs
|
||||
{
|
||||
public:
|
||||
InternalTextLogs(WriteBuffer & buf_out, bool color_) : wb(buf_out), color(color_) {}
|
||||
|
||||
|
||||
void write(const Block & block);
|
||||
/// Print internal server logs
|
||||
///
|
||||
/// Input blocks have to have the same structure as SystemLogsQueue::getSampleBlock():
|
||||
/// - event_time
|
||||
/// - event_time_microseconds
|
||||
/// - host_name
|
||||
/// - query_id
|
||||
/// - thread_id
|
||||
/// - priority
|
||||
/// - source
|
||||
/// - text
|
||||
void writeLogs(const Block & block);
|
||||
/// Print profile events.
|
||||
///
|
||||
/// Block:
|
||||
/// - host_name
|
||||
/// - current_time
|
||||
/// - thread_id
|
||||
/// - type
|
||||
/// - name
|
||||
/// - value
|
||||
///
|
||||
/// See also TCPHandler::sendProfileEvents() for block columns.
|
||||
void writeProfileEvents(const Block & block);
|
||||
|
||||
void flush()
|
||||
{
|
||||
|
||||
@ -267,19 +267,19 @@ bool LocalConnection::poll(size_t)
|
||||
}
|
||||
}
|
||||
|
||||
if (state->is_finished && send_progress && !state->sent_progress)
|
||||
{
|
||||
state->sent_progress = true;
|
||||
next_packet_type = Protocol::Server::Progress;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (state->is_finished)
|
||||
{
|
||||
finishQuery();
|
||||
return true;
|
||||
}
|
||||
|
||||
if (send_progress && !state->sent_progress)
|
||||
{
|
||||
state->sent_progress = true;
|
||||
next_packet_type = Protocol::Server::Progress;
|
||||
return true;
|
||||
}
|
||||
|
||||
if (state->block && state->block.value())
|
||||
{
|
||||
next_packet_type = Protocol::Server::Data;
|
||||
@ -293,7 +293,8 @@ bool LocalConnection::pollImpl()
|
||||
{
|
||||
Block block;
|
||||
auto next_read = pullBlock(block);
|
||||
if (block)
|
||||
|
||||
if (block && !state->io.null_format)
|
||||
{
|
||||
state->block.emplace(block);
|
||||
}
|
||||
|
||||
@ -77,3 +77,6 @@ target_link_libraries (average PRIVATE clickhouse_common_io)
|
||||
|
||||
add_executable (shell_command_inout shell_command_inout.cpp)
|
||||
target_link_libraries (shell_command_inout PRIVATE clickhouse_common_io)
|
||||
|
||||
add_executable (executable_udf executable_udf.cpp)
|
||||
target_link_libraries (executable_udf PRIVATE dbms)
|
||||
|
||||
44
src/Common/examples/executable_udf.cpp
Normal file
44
src/Common/examples/executable_udf.cpp
Normal file
@ -0,0 +1,44 @@
|
||||
#include <vector>
|
||||
#include <string>
|
||||
#include <iomanip>
|
||||
|
||||
#include <Common/SipHash.h>
|
||||
|
||||
#include <IO/ReadBufferFromFileDescriptor.h>
|
||||
#include <IO/WriteBufferFromFileDescriptor.h>
|
||||
#include <IO/ReadHelpers.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
#include <Common/Stopwatch.h>
|
||||
|
||||
using namespace DB;
|
||||
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
(void)(argc);
|
||||
(void)(argv);
|
||||
|
||||
std::string buffer;
|
||||
|
||||
ReadBufferFromFileDescriptor read_buffer(0);
|
||||
WriteBufferFromFileDescriptor write_buffer(1);
|
||||
size_t rows = 0;
|
||||
char dummy;
|
||||
|
||||
while (!read_buffer.eof()) {
|
||||
readIntText(rows, read_buffer);
|
||||
readChar(dummy, read_buffer);
|
||||
|
||||
for (size_t i = 0; i < rows; ++i) {
|
||||
readString(buffer, read_buffer);
|
||||
readChar(dummy, read_buffer);
|
||||
|
||||
writeString("Key ", write_buffer);
|
||||
writeString(buffer, write_buffer);
|
||||
writeChar('\n', write_buffer);
|
||||
}
|
||||
|
||||
write_buffer.next();
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
@ -118,7 +118,7 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p
|
||||
return absolute_path.starts_with(absolute_prefix_path);
|
||||
}
|
||||
|
||||
bool symlinkStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path)
|
||||
bool fileOrSymlinkPathStartsWith(const std::filesystem::path & path, const std::filesystem::path & prefix_path)
|
||||
{
|
||||
/// Differs from pathStartsWith in how `path` is normalized before comparison.
|
||||
/// Make `path` absolute if it was relative and put it into normalized form: remove
|
||||
@ -140,13 +140,14 @@ bool pathStartsWith(const String & path, const String & prefix_path)
|
||||
return pathStartsWith(filesystem_path, filesystem_prefix_path);
|
||||
}
|
||||
|
||||
bool symlinkStartsWith(const String & path, const String & prefix_path)
|
||||
bool fileOrSymlinkPathStartsWith(const String & path, const String & prefix_path)
|
||||
{
|
||||
auto filesystem_path = std::filesystem::path(path);
|
||||
auto filesystem_prefix_path = std::filesystem::path(prefix_path);
|
||||
|
||||
return symlinkStartsWith(filesystem_path, filesystem_prefix_path);
|
||||
return fileOrSymlinkPathStartsWith(filesystem_path, filesystem_prefix_path);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -35,8 +35,9 @@ bool pathStartsWith(const std::filesystem::path & path, const std::filesystem::p
|
||||
/// Returns true if path starts with prefix path
|
||||
bool pathStartsWith(const String & path, const String & prefix_path);
|
||||
|
||||
/// Returns true if symlink starts with prefix path
|
||||
bool symlinkStartsWith(const String & path, const String & prefix_path);
|
||||
/// Same as pathStartsWith, but without canonization, i.e. allowed to check symlinks.
|
||||
/// (Path is made absolute and normalized.)
|
||||
bool fileOrSymlinkPathStartsWith(const String & path, const String & prefix_path);
|
||||
|
||||
}
|
||||
|
||||
|
||||
@ -100,7 +100,7 @@ Pipe ExecutablePoolDictionarySource::getStreamForBlock(const Block & block)
|
||||
config.terminate_in_destructor_strategy = ShellCommand::DestructorStrategy{ true /*terminate_in_destructor*/, configuration.command_termination_timeout };
|
||||
auto shell_command = ShellCommand::execute(config);
|
||||
return shell_command;
|
||||
}, configuration.max_command_execution_time * 1000);
|
||||
}, configuration.max_command_execution_time * 10000);
|
||||
|
||||
if (!result)
|
||||
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED,
|
||||
|
||||
@ -31,7 +31,7 @@ FileDictionarySource::FileDictionarySource(
|
||||
, context(context_)
|
||||
{
|
||||
auto user_files_path = context->getUserFilesPath();
|
||||
if (created_from_ddl && !pathStartsWith(filepath, user_files_path))
|
||||
if (created_from_ddl && !fileOrSymlinkPathStartsWith(filepath, user_files_path))
|
||||
throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", filepath, user_files_path);
|
||||
}
|
||||
|
||||
|
||||
@ -41,13 +41,7 @@ LibraryDictionarySource::LibraryDictionarySource(
|
||||
, context(Context::createCopy(context_))
|
||||
{
|
||||
auto dictionaries_lib_path = context->getDictionariesLibPath();
|
||||
bool path_checked = false;
|
||||
if (fs::is_symlink(path))
|
||||
path_checked = symlinkStartsWith(path, dictionaries_lib_path);
|
||||
else
|
||||
path_checked = pathStartsWith(path, dictionaries_lib_path);
|
||||
|
||||
if (created_from_ddl && !path_checked)
|
||||
if (created_from_ddl && !fileOrSymlinkPathStartsWith(path, dictionaries_lib_path))
|
||||
throw Exception(ErrorCodes::PATH_ACCESS_DENIED, "File path {} is not inside {}", path, dictionaries_lib_path);
|
||||
|
||||
if (!fs::exists(path))
|
||||
|
||||
@ -42,7 +42,7 @@ FormatSettings getFormatSettings(ContextPtr context);
|
||||
template <typename T>
|
||||
FormatSettings getFormatSettings(ContextPtr context, const T & settings);
|
||||
|
||||
/** Allows to create an IBlockInputStream or IBlockOutputStream by the name of the format.
|
||||
/** Allows to create an IInputFormat or IOutputFormat by the name of the format.
|
||||
* Note: format and compression are independent things.
|
||||
*/
|
||||
class FormatFactory final : private boost::noncopyable
|
||||
|
||||
@ -56,7 +56,6 @@ NativeReader::NativeReader(ReadBuffer & istr_, UInt64 server_revision_,
|
||||
}
|
||||
}
|
||||
|
||||
// also resets few vars from IBlockInputStream (I didn't want to propagate resetParser upthere)
|
||||
void NativeReader::resetParser()
|
||||
{
|
||||
istr_concrete = nullptr;
|
||||
|
||||
@ -96,6 +96,9 @@ struct ReplaceRegexpImpl
|
||||
re2_st::StringPiece matches[max_captures];
|
||||
|
||||
size_t start_pos = 0;
|
||||
bool is_first_match = true;
|
||||
bool is_start_pos_added_one = false;
|
||||
|
||||
while (start_pos < static_cast<size_t>(input.length()))
|
||||
{
|
||||
/// If no more replacements possible for current string
|
||||
@ -103,6 +106,9 @@ struct ReplaceRegexpImpl
|
||||
|
||||
if (searcher.Match(input, start_pos, input.length(), re2_st::RE2::Anchor::UNANCHORED, matches, num_captures))
|
||||
{
|
||||
if (is_start_pos_added_one)
|
||||
start_pos -= 1;
|
||||
|
||||
const auto & match = matches[0];
|
||||
size_t bytes_to_copy = (match.data() - input.data()) - start_pos;
|
||||
|
||||
@ -112,6 +118,13 @@ struct ReplaceRegexpImpl
|
||||
res_offset += bytes_to_copy;
|
||||
start_pos += bytes_to_copy + match.length();
|
||||
|
||||
/// To avoid infinite loop.
|
||||
if (is_first_match && match.length() == 0 && !replace_one && input.length() > 1)
|
||||
{
|
||||
start_pos += 1;
|
||||
is_start_pos_added_one = true;
|
||||
}
|
||||
|
||||
/// Do substitution instructions
|
||||
for (const auto & it : instructions)
|
||||
{
|
||||
@ -129,8 +142,9 @@ struct ReplaceRegexpImpl
|
||||
}
|
||||
}
|
||||
|
||||
if (replace_one || match.length() == 0) /// Stop after match of zero length, to avoid infinite loop.
|
||||
if (replace_one || (!is_first_match && match.length() == 0))
|
||||
can_finish_current_string = true;
|
||||
is_first_match = false;
|
||||
}
|
||||
else
|
||||
can_finish_current_string = true;
|
||||
|
||||
@ -18,10 +18,10 @@ namespace ErrorCodes
|
||||
|
||||
|
||||
template <class DataTypeName, class Geometry, class Serializer, class NameHolder>
|
||||
class FunctionReadWkt : public IFunction
|
||||
class FunctionReadWKT : public IFunction
|
||||
{
|
||||
public:
|
||||
explicit FunctionReadWkt() = default;
|
||||
explicit FunctionReadWKT() = default;
|
||||
|
||||
static constexpr const char * name = NameHolder::name;
|
||||
|
||||
@ -72,36 +72,36 @@ public:
|
||||
|
||||
static FunctionPtr create(ContextPtr)
|
||||
{
|
||||
return std::make_shared<FunctionReadWkt<DataTypeName, Geometry, Serializer, NameHolder>>();
|
||||
return std::make_shared<FunctionReadWKT<DataTypeName, Geometry, Serializer, NameHolder>>();
|
||||
}
|
||||
};
|
||||
|
||||
struct ReadWktPointNameHolder
|
||||
struct ReadWKTPointNameHolder
|
||||
{
|
||||
static constexpr const char * name = "readWktPoint";
|
||||
static constexpr const char * name = "readWKTPoint";
|
||||
};
|
||||
|
||||
struct ReadWktRingNameHolder
|
||||
struct ReadWKTRingNameHolder
|
||||
{
|
||||
static constexpr const char * name = "readWktRing";
|
||||
static constexpr const char * name = "readWKTRing";
|
||||
};
|
||||
|
||||
struct ReadWktPolygonNameHolder
|
||||
struct ReadWKTPolygonNameHolder
|
||||
{
|
||||
static constexpr const char * name = "readWktPolygon";
|
||||
static constexpr const char * name = "readWKTPolygon";
|
||||
};
|
||||
|
||||
struct ReadWktMultiPolygonNameHolder
|
||||
struct ReadWKTMultiPolygonNameHolder
|
||||
{
|
||||
static constexpr const char * name = "readWktMultiPolygon";
|
||||
static constexpr const char * name = "readWKTMultiPolygon";
|
||||
};
|
||||
|
||||
void registerFunctionReadWkt(FunctionFactory & factory)
|
||||
void registerFunctionReadWKT(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionReadWkt<DataTypePointName, CartesianPoint, PointSerializer<CartesianPoint>, ReadWktPointNameHolder>>();
|
||||
factory.registerFunction<FunctionReadWkt<DataTypeRingName, CartesianRing, RingSerializer<CartesianPoint>, ReadWktRingNameHolder>>();
|
||||
factory.registerFunction<FunctionReadWkt<DataTypePolygonName, CartesianPolygon, PolygonSerializer<CartesianPoint>, ReadWktPolygonNameHolder>>();
|
||||
factory.registerFunction<FunctionReadWkt<DataTypeMultiPolygonName, CartesianMultiPolygon, MultiPolygonSerializer<CartesianPoint>, ReadWktMultiPolygonNameHolder>>();
|
||||
factory.registerFunction<FunctionReadWKT<DataTypePointName, CartesianPoint, PointSerializer<CartesianPoint>, ReadWKTPointNameHolder>>();
|
||||
factory.registerFunction<FunctionReadWKT<DataTypeRingName, CartesianRing, RingSerializer<CartesianPoint>, ReadWKTRingNameHolder>>();
|
||||
factory.registerFunction<FunctionReadWKT<DataTypePolygonName, CartesianPolygon, PolygonSerializer<CartesianPoint>, ReadWKTPolygonNameHolder>>();
|
||||
factory.registerFunction<FunctionReadWKT<DataTypeMultiPolygonName, CartesianMultiPolygon, MultiPolygonSerializer<CartesianPoint>, ReadWKTMultiPolygonNameHolder>>();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -23,7 +23,7 @@ void registerFunctionGeohashEncode(FunctionFactory & factory);
|
||||
void registerFunctionGeohashDecode(FunctionFactory & factory);
|
||||
void registerFunctionGeohashesInBox(FunctionFactory & factory);
|
||||
void registerFunctionWkt(FunctionFactory & factory);
|
||||
void registerFunctionReadWkt(FunctionFactory & factory);
|
||||
void registerFunctionReadWKT(FunctionFactory & factory);
|
||||
void registerFunctionSvg(FunctionFactory & factory);
|
||||
|
||||
#if USE_H3
|
||||
@ -79,7 +79,7 @@ void registerFunctionsGeo(FunctionFactory & factory)
|
||||
registerFunctionGeohashDecode(factory);
|
||||
registerFunctionGeohashesInBox(factory);
|
||||
registerFunctionWkt(factory);
|
||||
registerFunctionReadWkt(factory);
|
||||
registerFunctionReadWKT(factory);
|
||||
registerFunctionSvg(factory);
|
||||
|
||||
#if USE_H3
|
||||
|
||||
@ -102,6 +102,7 @@ public:
|
||||
void registerFunctionSvg(FunctionFactory & factory)
|
||||
{
|
||||
factory.registerFunction<FunctionSvg>();
|
||||
factory.registerAlias("SVG", "svg");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@ -121,7 +121,7 @@ struct Progress
|
||||
|
||||
|
||||
/** Callback to track the progress of the query.
|
||||
* Used in IBlockInputStream and Context.
|
||||
* Used in QueryPipeline and Context.
|
||||
* The function takes the number of rows in the last block, the number of bytes in the last block.
|
||||
* Note that the callback can be called from different threads.
|
||||
*/
|
||||
|
||||
@ -43,8 +43,6 @@ namespace ErrorCodes
|
||||
extern const int UNKNOWN_AGGREGATED_DATA_VARIANT;
|
||||
}
|
||||
|
||||
class IBlockOutputStream;
|
||||
|
||||
/** Different data structures that can be used for aggregation
|
||||
* For efficiency, the aggregation data itself is put into the pool.
|
||||
* Data and pool ownership (states of aggregate functions)
|
||||
|
||||
@ -2971,8 +2971,12 @@ void Context::setAsynchronousInsertQueue(const std::shared_ptr<AsynchronousInser
|
||||
shared->async_insert_queue = ptr;
|
||||
}
|
||||
|
||||
void Context::initializeBackgroundExecutors()
|
||||
void Context::initializeBackgroundExecutorsIfNeeded()
|
||||
{
|
||||
auto lock = getLock();
|
||||
if (is_background_executors_initialized)
|
||||
return;
|
||||
|
||||
const size_t max_merges_and_mutations = getSettingsRef().background_pool_size * getSettingsRef().background_merges_mutations_concurrency_ratio;
|
||||
|
||||
/// With this executor we can execute more tasks than threads we have
|
||||
@ -3019,6 +3023,8 @@ void Context::initializeBackgroundExecutors()
|
||||
|
||||
LOG_INFO(shared->log, "Initialized background executor for common operations (e.g. clearing old parts) with num_threads={}, num_tasks={}",
|
||||
getSettingsRef().background_common_pool_size, getSettingsRef().background_common_pool_size);
|
||||
|
||||
is_background_executors_initialized = true;
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -293,6 +293,8 @@ private:
|
||||
|
||||
/// A flag, used to distinguish between user query and internal query to a database engine (MaterializedPostgreSQL).
|
||||
bool is_internal_query = false;
|
||||
/// Has initializeBackgroundExecutors() method been executed?
|
||||
bool is_background_executors_initialized = false;
|
||||
|
||||
|
||||
public:
|
||||
@ -636,13 +638,13 @@ public:
|
||||
const Settings & getSettingsRef() const { return settings; }
|
||||
|
||||
void setProgressCallback(ProgressCallback callback);
|
||||
/// Used in InterpreterSelectQuery to pass it to the IBlockInputStream.
|
||||
/// Used in executeQuery() to pass it to the QueryPipeline.
|
||||
ProgressCallback getProgressCallback() const;
|
||||
|
||||
void setFileProgressCallback(FileProgressCallback && callback) { file_progress_callback = callback; }
|
||||
FileProgressCallback getFileProgressCallback() const { return file_progress_callback; }
|
||||
|
||||
/** Set in executeQuery and InterpreterSelectQuery. Then it is used in IBlockInputStream,
|
||||
/** Set in executeQuery and InterpreterSelectQuery. Then it is used in QueryPipeline,
|
||||
* to update and monitor information about the total number of resources spent for the query.
|
||||
*/
|
||||
void setProcessListElement(QueryStatus * elem);
|
||||
@ -867,7 +869,7 @@ public:
|
||||
void setReadTaskCallback(ReadTaskCallback && callback);
|
||||
|
||||
/// Background executors related methods
|
||||
void initializeBackgroundExecutors();
|
||||
void initializeBackgroundExecutorsIfNeeded();
|
||||
|
||||
MergeMutateBackgroundExecutorPtr getMergeMutateExecutor() const;
|
||||
OrdinaryBackgroundExecutorPtr getMovesExecutor() const;
|
||||
|
||||
@ -11,6 +11,11 @@
|
||||
namespace ProfileEvents
|
||||
{
|
||||
|
||||
std::shared_ptr<DB::DataTypeEnum8> TypeEnum = std::make_shared<DB::DataTypeEnum8>(DB::DataTypeEnum8::Values{
|
||||
{ "increment", static_cast<Int8>(INCREMENT)},
|
||||
{ "gauge", static_cast<Int8>(GAUGE)},
|
||||
});
|
||||
|
||||
/// Put implementation here to avoid extra linking dependencies for clickhouse_common_io
|
||||
void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, bool nonzero_only)
|
||||
{
|
||||
|
||||
@ -1,5 +1,6 @@
|
||||
#pragma once
|
||||
#include <Common/ProfileEvents.h>
|
||||
#include <DataTypes/DataTypeEnum.h>
|
||||
#include <Columns/IColumn.h>
|
||||
|
||||
|
||||
@ -9,4 +10,13 @@ namespace ProfileEvents
|
||||
/// Dumps profile events to columns Map(String, UInt64)
|
||||
void dumpToMapColumn(const Counters::Snapshot & counters, DB::IColumn * column, bool nonzero_only = true);
|
||||
|
||||
/// This is for ProfileEvents packets.
|
||||
enum Type : int8_t
|
||||
{
|
||||
INCREMENT = 1,
|
||||
GAUGE = 2,
|
||||
};
|
||||
|
||||
extern std::shared_ptr<DB::DataTypeEnum8> TypeEnum;
|
||||
|
||||
}
|
||||
|
||||
@ -161,7 +161,7 @@ void loadMetadata(ContextMutablePtr context, const String & default_database_nam
|
||||
bool create_default_db_if_not_exists = !default_database_name.empty();
|
||||
bool metadata_dir_for_default_db_already_exists = databases.count(default_database_name);
|
||||
if (create_default_db_if_not_exists && !metadata_dir_for_default_db_already_exists)
|
||||
databases.emplace(default_database_name, path + "/" + escapeForFileName(default_database_name));
|
||||
databases.emplace(default_database_name, std::filesystem::path(path) / escapeForFileName(default_database_name));
|
||||
|
||||
TablesLoader::Databases loaded_databases;
|
||||
for (const auto & [name, db_path] : databases)
|
||||
|
||||
@ -72,7 +72,8 @@ public:
|
||||
|
||||
InputPort & getPort(PortKind kind) { return *std::next(inputs.begin(), kind); }
|
||||
|
||||
/// Compatible to IBlockOutputStream interface
|
||||
/// Compatibility with old interface.
|
||||
/// TODO: separate formats and processors.
|
||||
|
||||
void write(const Block & block);
|
||||
|
||||
|
||||
@ -69,8 +69,7 @@ void SourceWithProgress::work()
|
||||
}
|
||||
}
|
||||
|
||||
/// Aggregated copy-paste from IBlockInputStream::progressImpl.
|
||||
/// Most of this must be done in PipelineExecutor outside. Now it's done for compatibility with IBlockInputStream.
|
||||
/// TODO: Most of this must be done in PipelineExecutor outside.
|
||||
void SourceWithProgress::progress(const Progress & value)
|
||||
{
|
||||
was_progress_called = true;
|
||||
@ -135,14 +134,12 @@ void SourceWithProgress::progress(const Progress & value)
|
||||
|
||||
if (last_profile_events_update_time + profile_events_update_period_microseconds < total_elapsed_microseconds)
|
||||
{
|
||||
/// Should be done in PipelineExecutor.
|
||||
/// It is here for compatibility with IBlockInputsStream.
|
||||
/// TODO: Should be done in PipelineExecutor.
|
||||
CurrentThread::updatePerformanceCounters();
|
||||
last_profile_events_update_time = total_elapsed_microseconds;
|
||||
}
|
||||
|
||||
/// Should be done in PipelineExecutor.
|
||||
/// It is here for compatibility with IBlockInputsStream.
|
||||
/// TODO: Should be done in PipelineExecutor.
|
||||
limits.speed_limits.throttle(progress.read_rows, progress.read_bytes, total_rows, total_elapsed_microseconds);
|
||||
|
||||
if (quota && limits.mode == LimitsMode::LIMITS_TOTAL)
|
||||
|
||||
@ -12,7 +12,7 @@ class Block;
|
||||
class ReadBuffer;
|
||||
class WriteBuffer;
|
||||
|
||||
/// Information for profiling. See IBlockInputStream.h
|
||||
/// Information for profiling. See SourceWithProgress.h
|
||||
struct ProfileInfo
|
||||
{
|
||||
bool started = false;
|
||||
|
||||
@ -129,7 +129,6 @@ public:
|
||||
void setLeafLimits(const SizeLimits & limits) { pipe.setLeafLimits(limits); }
|
||||
void setQuota(const std::shared_ptr<const EnabledQuota> & quota) { pipe.setQuota(quota); }
|
||||
|
||||
/// For compatibility with IBlockInputStream.
|
||||
void setProgressCallback(const ProgressCallback & callback);
|
||||
void setProcessListElement(QueryStatus * elem);
|
||||
|
||||
|
||||
@ -30,6 +30,7 @@
|
||||
#include <Interpreters/InternalTextLogsQueue.h>
|
||||
#include <Interpreters/OpenTelemetrySpanLog.h>
|
||||
#include <Interpreters/Session.h>
|
||||
#include <Interpreters/ProfileEventsExt.h>
|
||||
#include <Storages/StorageReplicatedMergeTree.h>
|
||||
#include <Storages/MergeTree/MergeTreeDataPartUUID.h>
|
||||
#include <Storages/StorageS3Cluster.h>
|
||||
@ -831,12 +832,6 @@ namespace
|
||||
{
|
||||
using namespace ProfileEvents;
|
||||
|
||||
enum ProfileEventTypes : int8_t
|
||||
{
|
||||
INCREMENT = 1,
|
||||
GAUGE = 2,
|
||||
};
|
||||
|
||||
constexpr size_t NAME_COLUMN_INDEX = 4;
|
||||
constexpr size_t VALUE_COLUMN_INDEX = 5;
|
||||
|
||||
@ -879,7 +874,7 @@ namespace
|
||||
columns[i++]->insertData(host_name.data(), host_name.size());
|
||||
columns[i++]->insert(UInt64(snapshot.current_time));
|
||||
columns[i++]->insert(UInt64{snapshot.thread_id});
|
||||
columns[i++]->insert(ProfileEventTypes::INCREMENT);
|
||||
columns[i++]->insert(ProfileEvents::Type::INCREMENT);
|
||||
}
|
||||
}
|
||||
|
||||
@ -893,7 +888,7 @@ namespace
|
||||
columns[i++]->insertData(host_name.data(), host_name.size());
|
||||
columns[i++]->insert(UInt64(snapshot.current_time));
|
||||
columns[i++]->insert(UInt64{snapshot.thread_id});
|
||||
columns[i++]->insert(ProfileEventTypes::GAUGE);
|
||||
columns[i++]->insert(ProfileEvents::Type::GAUGE);
|
||||
|
||||
columns[i++]->insertData(MemoryTracker::USAGE_EVENT_NAME, strlen(MemoryTracker::USAGE_EVENT_NAME));
|
||||
columns[i++]->insert(snapshot.memory_usage);
|
||||
@ -907,18 +902,11 @@ void TCPHandler::sendProfileEvents()
|
||||
if (client_tcp_protocol_version < DBMS_MIN_PROTOCOL_VERSION_WITH_PROFILE_EVENTS)
|
||||
return;
|
||||
|
||||
auto profile_event_type = std::make_shared<DataTypeEnum8>(
|
||||
DataTypeEnum8::Values
|
||||
{
|
||||
{ "increment", static_cast<Int8>(INCREMENT)},
|
||||
{ "gauge", static_cast<Int8>(GAUGE)},
|
||||
});
|
||||
|
||||
NamesAndTypesList column_names_and_types = {
|
||||
{ "host_name", std::make_shared<DataTypeString>() },
|
||||
{ "current_time", std::make_shared<DataTypeDateTime>() },
|
||||
{ "thread_id", std::make_shared<DataTypeUInt64>() },
|
||||
{ "type", profile_event_type },
|
||||
{ "type", ProfileEvents::TypeEnum },
|
||||
{ "name", std::make_shared<DataTypeString>() },
|
||||
{ "value", std::make_shared<DataTypeUInt64>() },
|
||||
};
|
||||
|
||||
@ -205,6 +205,8 @@ MergeTreeData::MergeTreeData(
|
||||
, background_operations_assignee(*this, BackgroundJobsAssignee::Type::DataProcessing, getContext())
|
||||
, background_moves_assignee(*this, BackgroundJobsAssignee::Type::Moving, getContext())
|
||||
{
|
||||
context_->getGlobalContext()->initializeBackgroundExecutorsIfNeeded();
|
||||
|
||||
const auto settings = getSettings();
|
||||
allow_nullable_key = attach || settings->allow_nullable_key;
|
||||
|
||||
|
||||
@ -160,9 +160,10 @@ size_t MergeTreeReaderCompact::readRows(size_t from_mark, bool continue_reading,
|
||||
readData(column_from_part, column, from_mark, *column_positions[pos], rows_to_read, read_only_offsets[pos]);
|
||||
|
||||
size_t read_rows_in_column = column->size() - column_size_before_reading;
|
||||
if (read_rows_in_column < rows_to_read)
|
||||
throw Exception("Cannot read all data in MergeTreeReaderCompact. Rows read: " + toString(read_rows_in_column) +
|
||||
". Rows expected: " + toString(rows_to_read) + ".", ErrorCodes::CANNOT_READ_ALL_DATA);
|
||||
if (read_rows_in_column != rows_to_read)
|
||||
throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA,
|
||||
"Cannot read all data in MergeTreeReaderCompact. Rows read: {}. Rows expected: {}.",
|
||||
read_rows_in_column, rows_to_read);
|
||||
}
|
||||
catch (Exception & e)
|
||||
{
|
||||
|
||||
@ -10,6 +10,7 @@ namespace DB
|
||||
namespace ErrorCodes
|
||||
{
|
||||
extern const int ARGUMENT_OUT_OF_BOUND;
|
||||
extern const int CANNOT_READ_ALL_DATA;
|
||||
}
|
||||
|
||||
|
||||
@ -76,6 +77,10 @@ MergeTreeReaderStream::MergeTreeReaderStream(
|
||||
if (max_mark_range_bytes != 0)
|
||||
read_settings = read_settings.adjustBufferSize(max_mark_range_bytes);
|
||||
|
||||
//// Empty buffer does not makes progress.
|
||||
if (!read_settings.local_fs_buffer_size || !read_settings.remote_fs_buffer_size)
|
||||
throw Exception(ErrorCodes::CANNOT_READ_ALL_DATA, "Cannot read to empty buffer.");
|
||||
|
||||
/// Initialize the objects that shall be used to perform read operations.
|
||||
if (uncompressed_cache)
|
||||
{
|
||||
|
||||
@ -69,10 +69,6 @@ size_t MergeTreeReaderWide::readRows(size_t from_mark, bool continue_reading, si
|
||||
size_t num_columns = columns.size();
|
||||
checkNumberOfColumns(num_columns);
|
||||
|
||||
/// Pointers to offset columns that are common to the nested data structure columns.
|
||||
/// If append is true, then the value will be equal to nullptr and will be used only to
|
||||
/// check that the offsets column has been already read.
|
||||
OffsetColumns offset_columns;
|
||||
std::unordered_map<String, ISerialization::SubstreamsCache> caches;
|
||||
|
||||
std::unordered_set<std::string> prefetched_streams;
|
||||
|
||||
@ -3,6 +3,8 @@
|
||||
#include <filesystem>
|
||||
|
||||
#include <Common/ShellCommand.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
|
||||
#include <Core/Block.h>
|
||||
|
||||
#include <IO/ReadHelpers.h>
|
||||
@ -111,9 +113,16 @@ Pipe StorageExecutable::read(
|
||||
{
|
||||
auto user_scripts_path = context->getUserScriptsPath();
|
||||
auto script_path = user_scripts_path + '/' + script_name;
|
||||
if (!std::filesystem::exists(std::filesystem::path(script_path)))
|
||||
|
||||
if (!pathStartsWith(script_path, user_scripts_path))
|
||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
||||
"Executable file {} does not exists inside {}",
|
||||
"Executable file {} must be inside user scripts folder {}",
|
||||
script_name,
|
||||
user_scripts_path);
|
||||
|
||||
if (!std::filesystem::exists(std::filesystem::path(script_path)))
|
||||
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
|
||||
"Executable file {} does not exist inside user scripts folder {}",
|
||||
script_name,
|
||||
user_scripts_path);
|
||||
|
||||
@ -139,9 +148,9 @@ Pipe StorageExecutable::read(
|
||||
bool result = process_pool->tryBorrowObject(process, [&config, this]()
|
||||
{
|
||||
config.terminate_in_destructor_strategy = ShellCommand::DestructorStrategy{ true /*terminate_in_destructor*/, settings.command_termination_timeout };
|
||||
auto shell_command = ShellCommand::execute(config);
|
||||
auto shell_command = ShellCommand::executeDirect(config);
|
||||
return shell_command;
|
||||
}, settings.max_command_execution_time * 1000);
|
||||
}, settings.max_command_execution_time * 10000);
|
||||
|
||||
if (!result)
|
||||
throw Exception(ErrorCodes::TIMEOUT_EXCEEDED,
|
||||
|
||||
@ -22,6 +22,7 @@
|
||||
#include <Common/escapeForFileName.h>
|
||||
#include <Common/typeid_cast.h>
|
||||
#include <Common/parseGlobs.h>
|
||||
#include <Common/filesystemHelpers.h>
|
||||
#include <Storages/ColumnsDescription.h>
|
||||
#include <Storages/StorageInMemoryMetadata.h>
|
||||
|
||||
@ -124,8 +125,8 @@ void checkCreationIsAllowed(ContextPtr context_global, const std::string & db_di
|
||||
return;
|
||||
|
||||
/// "/dev/null" is allowed for perf testing
|
||||
if (!startsWith(table_path, db_dir_path) && table_path != "/dev/null")
|
||||
throw Exception("File is not inside " + db_dir_path, ErrorCodes::DATABASE_ACCESS_DENIED);
|
||||
if (!fileOrSymlinkPathStartsWith(table_path, db_dir_path) && table_path != "/dev/null")
|
||||
throw Exception(ErrorCodes::DATABASE_ACCESS_DENIED, "File `{}` is not inside `{}`", table_path, db_dir_path);
|
||||
|
||||
if (fs::exists(table_path) && fs::is_directory(table_path))
|
||||
throw Exception("File must not be a directory", ErrorCodes::INCORRECT_FILE_NAME);
|
||||
@ -140,7 +141,10 @@ Strings StorageFile::getPathsList(const String & table_path, const String & user
|
||||
fs_table_path = user_files_absolute_path / fs_table_path;
|
||||
|
||||
Strings paths;
|
||||
const String path = fs::weakly_canonical(fs_table_path);
|
||||
/// Do not use fs::canonical or fs::weakly_canonical.
|
||||
/// Otherwise it will not allow to work with symlinks in `user_files_path` directory.
|
||||
String path = fs::absolute(fs_table_path);
|
||||
path = fs::path(path).lexically_normal(); /// Normalize path.
|
||||
if (path.find_first_of("*?{") == std::string::npos)
|
||||
{
|
||||
std::error_code error;
|
||||
|
||||
51
tests/ci/compress_files.py
Normal file
51
tests/ci/compress_files.py
Normal file
@ -0,0 +1,51 @@
|
||||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
import logging
|
||||
import os
|
||||
|
||||
def compress_file_fast(path, archive_path):
|
||||
if os.path.exists('/usr/bin/pigz'):
|
||||
subprocess.check_call("pigz < {} > {}".format(path, archive_path), shell=True)
|
||||
else:
|
||||
subprocess.check_call("gzip < {} > {}".format(path, archive_path), shell=True)
|
||||
|
||||
|
||||
def compress_fast(path, archive_path, exclude=None):
|
||||
pigz_part = ''
|
||||
if os.path.exists('/usr/bin/pigz'):
|
||||
logging.info("pigz found, will compress and decompress faster")
|
||||
pigz_part = "--use-compress-program='pigz'"
|
||||
else:
|
||||
pigz_part = '-z'
|
||||
logging.info("no pigz, compressing with default tar")
|
||||
|
||||
if exclude is None:
|
||||
exclude_part = ""
|
||||
elif isinstance(exclude, list):
|
||||
exclude_part = " ".join(["--exclude {}".format(x) for x in exclude])
|
||||
else:
|
||||
exclude_part = "--exclude {}".format(str(exclude))
|
||||
|
||||
fname = os.path.basename(path)
|
||||
if os.path.isfile(path):
|
||||
path = os.path.dirname(path)
|
||||
else:
|
||||
path += "/.."
|
||||
cmd = "tar {} {} -cf {} -C {} {}".format(pigz_part, exclude_part, archive_path, path, fname)
|
||||
logging.debug("compress_fast cmd:{}".format(cmd))
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
|
||||
def decompress_fast(archive_path, result_path=None):
|
||||
pigz_part = ''
|
||||
if os.path.exists('/usr/bin/pigz'):
|
||||
logging.info("pigz found, will compress and decompress faster ('{}' -> '{}')".format(archive_path, result_path))
|
||||
pigz_part = "--use-compress-program='pigz'"
|
||||
else:
|
||||
pigz_part = '-z'
|
||||
logging.info("no pigz, decompressing with default tar ('{}' -> '{}')".format(archive_path, result_path))
|
||||
|
||||
if result_path is None:
|
||||
subprocess.check_call("tar {} -xf {}".format(pigz_part, archive_path), shell=True)
|
||||
else:
|
||||
subprocess.check_call("tar {} -xf {} -C {}".format(pigz_part, archive_path, result_path), shell=True)
|
||||
231
tests/ci/docker_images_check.py
Normal file
231
tests/ci/docker_images_check.py
Normal file
@ -0,0 +1,231 @@
|
||||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
import logging
|
||||
from report import create_test_html_report
|
||||
from s3_helper import S3Helper
|
||||
import json
|
||||
import os
|
||||
from pr_info import PRInfo
|
||||
from github import Github
|
||||
import shutil
|
||||
from get_robot_token import get_best_robot_token, get_parameter_from_ssm
|
||||
|
||||
NAME = "Push to Dockerhub (actions)"
|
||||
|
||||
def get_changed_docker_images(pr_info, repo_path, image_file_path):
|
||||
images_dict = {}
|
||||
path_to_images_file = os.path.join(repo_path, image_file_path)
|
||||
if os.path.exists(path_to_images_file):
|
||||
with open(path_to_images_file, 'r') as dict_file:
|
||||
images_dict = json.load(dict_file)
|
||||
else:
|
||||
logging.info("Image file %s doesnt exists in repo %s", image_file_path, repo_path)
|
||||
|
||||
dockerhub_repo_name = 'yandex'
|
||||
if not images_dict:
|
||||
return [], dockerhub_repo_name
|
||||
|
||||
files_changed = pr_info.changed_files
|
||||
|
||||
logging.info("Changed files for PR %s @ %s: %s", pr_info.number, pr_info.sha, str(files_changed))
|
||||
|
||||
changed_images = []
|
||||
|
||||
for dockerfile_dir, image_description in images_dict.items():
|
||||
if image_description['name'].startswith('clickhouse/'):
|
||||
dockerhub_repo_name = 'clickhouse'
|
||||
|
||||
for f in files_changed:
|
||||
if f.startswith(dockerfile_dir):
|
||||
logging.info(
|
||||
"Found changed file '%s' which affects docker image '%s' with path '%s'",
|
||||
f, image_description['name'], dockerfile_dir)
|
||||
changed_images.append(dockerfile_dir)
|
||||
break
|
||||
|
||||
# The order is important: dependents should go later than bases, so that
|
||||
# they are built with updated base versions.
|
||||
index = 0
|
||||
while index < len(changed_images):
|
||||
image = changed_images[index]
|
||||
for dependent in images_dict[image]['dependent']:
|
||||
logging.info(
|
||||
"Marking docker image '%s' as changed because it depends on changed docker image '%s'",
|
||||
dependent, image)
|
||||
changed_images.append(dependent)
|
||||
index += 1
|
||||
if index > 100:
|
||||
# Sanity check to prevent infinite loop.
|
||||
raise "Too many changed docker images, this is a bug." + str(changed_images)
|
||||
|
||||
# If a dependent image was already in the list because its own files
|
||||
# changed, but then it was added as a dependent of a changed base, we
|
||||
# must remove the earlier entry so that it doesn't go earlier than its
|
||||
# base. This way, the dependent will be rebuilt later than the base, and
|
||||
# will correctly use the updated version of the base.
|
||||
seen = set()
|
||||
no_dups_reversed = []
|
||||
for x in reversed(changed_images):
|
||||
if x not in seen:
|
||||
seen.add(x)
|
||||
no_dups_reversed.append(x)
|
||||
|
||||
result = [(x, images_dict[x]['name']) for x in reversed(no_dups_reversed)]
|
||||
logging.info("Changed docker images for PR %s @ %s: '%s'", pr_info.number, pr_info.sha, result)
|
||||
return result, dockerhub_repo_name
|
||||
|
||||
def build_and_push_one_image(path_to_dockerfile_folder, image_name, version_string):
|
||||
logging.info("Building docker image %s with version %s from path %s", image_name, version_string, path_to_dockerfile_folder)
|
||||
build_log = None
|
||||
push_log = None
|
||||
with open('build_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
|
||||
cmd = "docker build --network=host -t {im}:{ver} {path}".format(im=image_name, ver=version_string, path=path_to_dockerfile_folder)
|
||||
retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
|
||||
build_log = str(pl.name)
|
||||
if retcode != 0:
|
||||
return False, build_log, None
|
||||
|
||||
with open('tag_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
|
||||
cmd = "docker build --network=host -t {im} {path}".format(im=image_name, path=path_to_dockerfile_folder)
|
||||
retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
|
||||
build_log = str(pl.name)
|
||||
if retcode != 0:
|
||||
return False, build_log, None
|
||||
|
||||
logging.info("Pushing image %s to dockerhub", image_name)
|
||||
|
||||
with open('push_log_' + str(image_name).replace('/', '_') + "_" + version_string, 'w') as pl:
|
||||
cmd = "docker push {im}:{ver}".format(im=image_name, ver=version_string)
|
||||
retcode = subprocess.Popen(cmd, shell=True, stderr=pl, stdout=pl).wait()
|
||||
push_log = str(pl.name)
|
||||
if retcode != 0:
|
||||
return False, build_log, push_log
|
||||
|
||||
logging.info("Processing of %s successfully finished", image_name)
|
||||
return True, build_log, push_log
|
||||
|
||||
def process_single_image(versions, path_to_dockerfile_folder, image_name):
|
||||
logging.info("Image will be pushed with versions %s", ', '.join(versions))
|
||||
result = []
|
||||
for ver in versions:
|
||||
for i in range(5):
|
||||
success, build_log, push_log = build_and_push_one_image(path_to_dockerfile_folder, image_name, ver)
|
||||
if success:
|
||||
result.append((image_name + ":" + ver, build_log, push_log, 'OK'))
|
||||
break
|
||||
logging.info("Got error will retry %s time and sleep for %s seconds", i, i * 5)
|
||||
time.sleep(i * 5)
|
||||
else:
|
||||
result.append((image_name + ":" + ver, build_log, push_log, 'FAIL'))
|
||||
|
||||
logging.info("Processing finished")
|
||||
return result
|
||||
|
||||
|
||||
def process_test_results(s3_client, test_results, s3_path_prefix):
|
||||
overall_status = 'success'
|
||||
processed_test_results = []
|
||||
for image, build_log, push_log, status in test_results:
|
||||
if status != 'OK':
|
||||
overall_status = 'failure'
|
||||
url_part = ''
|
||||
if build_log is not None and os.path.exists(build_log):
|
||||
build_url = s3_client.upload_test_report_to_s3(
|
||||
build_log,
|
||||
s3_path_prefix + "/" + os.path.basename(build_log))
|
||||
url_part += '<a href="{}">build_log</a>'.format(build_url)
|
||||
if push_log is not None and os.path.exists(push_log):
|
||||
push_url = s3_client.upload_test_report_to_s3(
|
||||
push_log,
|
||||
s3_path_prefix + "/" + os.path.basename(push_log))
|
||||
if url_part:
|
||||
url_part += ', '
|
||||
url_part += '<a href="{}">push_log</a>'.format(push_url)
|
||||
if url_part:
|
||||
test_name = image + ' (' + url_part + ')'
|
||||
else:
|
||||
test_name = image
|
||||
processed_test_results.append((test_name, status))
|
||||
return overall_status, processed_test_results
|
||||
|
||||
def upload_results(s3_client, pr_number, commit_sha, test_results):
|
||||
s3_path_prefix = f"{pr_number}/{commit_sha}/" + NAME.lower().replace(' ', '_')
|
||||
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
|
||||
branch_name = "master"
|
||||
if pr_number != 0:
|
||||
branch_name = "PR #{}".format(pr_number)
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
|
||||
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
|
||||
|
||||
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
|
||||
html_report = create_test_html_report(NAME, test_results, "https://hub.docker.com/u/clickhouse", task_url, branch_url, branch_name, commit_url)
|
||||
with open('report.html', 'w') as f:
|
||||
f.write(html_report)
|
||||
|
||||
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
|
||||
logging.info("Search result in url %s", url)
|
||||
return url
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
repo_path = os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../"))
|
||||
temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'docker_images_check')
|
||||
dockerhub_password = get_parameter_from_ssm('dockerhub_robot_password')
|
||||
|
||||
if os.path.exists(temp_path):
|
||||
shutil.rmtree(temp_path)
|
||||
|
||||
if not os.path.exists(temp_path):
|
||||
os.makedirs(temp_path)
|
||||
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
|
||||
pr_info = PRInfo(event, False, True)
|
||||
changed_images, dockerhub_repo_name = get_changed_docker_images(pr_info, repo_path, "docker/images.json")
|
||||
logging.info("Has changed images %s", ', '.join([str(image[0]) for image in changed_images]))
|
||||
pr_commit_version = str(pr_info.number) + '-' + pr_info.sha
|
||||
|
||||
versions = [str(pr_info.number), pr_commit_version]
|
||||
|
||||
subprocess.check_output("docker login --username 'robotclickhouse' --password '{}'".format(dockerhub_password), shell=True)
|
||||
|
||||
result_images = {}
|
||||
images_processing_result = []
|
||||
for rel_path, image_name in changed_images:
|
||||
full_path = os.path.join(repo_path, rel_path)
|
||||
images_processing_result += process_single_image(versions, full_path, image_name)
|
||||
result_images[image_name] = pr_commit_version
|
||||
|
||||
if len(changed_images):
|
||||
description = "Updated " + ','.join([im[1] for im in changed_images])
|
||||
else:
|
||||
description = "Nothing to update"
|
||||
|
||||
|
||||
if len(description) >= 140:
|
||||
description = description[:136] + "..."
|
||||
|
||||
s3_helper = S3Helper('https://s3.amazonaws.com')
|
||||
|
||||
s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_')
|
||||
status, test_results = process_test_results(s3_helper, images_processing_result, s3_path_prefix)
|
||||
|
||||
url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results)
|
||||
|
||||
gh = Github(get_best_robot_token())
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=status, target_url=url)
|
||||
|
||||
with open(os.path.join(temp_path, 'changed_images.json'), 'w') as images_file:
|
||||
json.dump(result_images, images_file)
|
||||
|
||||
print("::notice ::Report url: {}".format(url))
|
||||
print("::set-output name=url_output::\"{}\"".format(url))
|
||||
44
tests/ci/finish_check.py
Normal file
44
tests/ci/finish_check.py
Normal file
@ -0,0 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
import logging
|
||||
from github import Github
|
||||
from pr_info import PRInfo
|
||||
import json
|
||||
import os
|
||||
from get_robot_token import get_best_robot_token
|
||||
|
||||
NAME = 'Run Check (actions)'
|
||||
|
||||
def filter_statuses(statuses):
|
||||
"""
|
||||
Squash statuses to latest state
|
||||
1. context="first", state="success", update_time=1
|
||||
2. context="second", state="success", update_time=2
|
||||
3. context="first", stat="failure", update_time=3
|
||||
=========>
|
||||
1. context="second", state="success"
|
||||
2. context="first", stat="failure"
|
||||
"""
|
||||
filt = {}
|
||||
for status in sorted(statuses, key=lambda x: x.updated_at):
|
||||
filt[status.context] = status
|
||||
return filt
|
||||
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
|
||||
pr_info = PRInfo(event, need_orgs=True)
|
||||
gh = Github(get_best_robot_token())
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
|
||||
url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
statuses = filter_statuses(list(commit.get_statuses()))
|
||||
if NAME in statuses and statuses[NAME].state == "pending":
|
||||
commit.create_status(context=NAME, description="All checks finished", state="success", target_url=url)
|
||||
20
tests/ci/get_robot_token.py
Normal file
20
tests/ci/get_robot_token.py
Normal file
@ -0,0 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
import boto3
|
||||
from github import Github
|
||||
|
||||
def get_parameter_from_ssm(name, decrypt=True, client=None):
|
||||
if not client:
|
||||
client = boto3.client('ssm', region_name='us-east-1')
|
||||
return client.get_parameter(Name=name, WithDecryption=decrypt)['Parameter']['Value']
|
||||
|
||||
def get_best_robot_token(token_prefix_env_name="github_robot_token_", total_tokens=4):
|
||||
client = boto3.client('ssm', region_name='us-east-1')
|
||||
tokens = {}
|
||||
for i in range(1, total_tokens + 1):
|
||||
token_name = token_prefix_env_name + str(i)
|
||||
token = get_parameter_from_ssm(token_name, True, client)
|
||||
gh = Github(token)
|
||||
rest, _ = gh.rate_limiting
|
||||
tokens[token] = rest
|
||||
|
||||
return max(tokens.items(), key=lambda x: x[1])[0]
|
||||
13
tests/ci/metrics_lambda/Dockerfile
Normal file
13
tests/ci/metrics_lambda/Dockerfile
Normal file
@ -0,0 +1,13 @@
|
||||
FROM public.ecr.aws/lambda/python:3.9
|
||||
|
||||
# Copy function code
|
||||
COPY app.py ${LAMBDA_TASK_ROOT}
|
||||
|
||||
# Install the function's dependencies using file requirements.txt
|
||||
# from your project folder.
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
|
||||
|
||||
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
|
||||
CMD [ "app.handler" ]
|
||||
143
tests/ci/metrics_lambda/app.py
Normal file
143
tests/ci/metrics_lambda/app.py
Normal file
@ -0,0 +1,143 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
import argparse
|
||||
import jwt
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
from collections import namedtuple
|
||||
|
||||
def get_key_and_app_from_aws():
|
||||
import boto3
|
||||
secret_name = "clickhouse_github_secret_key"
|
||||
session = boto3.session.Session()
|
||||
client = session.client(
|
||||
service_name='secretsmanager',
|
||||
)
|
||||
get_secret_value_response = client.get_secret_value(
|
||||
SecretId=secret_name
|
||||
)
|
||||
data = json.loads(get_secret_value_response['SecretString'])
|
||||
return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
|
||||
|
||||
def handler(event, context):
|
||||
private_key, app_id = get_key_and_app_from_aws()
|
||||
main(private_key, app_id, True)
|
||||
|
||||
def get_installation_id(jwt_token):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.get("https://api.github.com/app/installations", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data[0]['id']
|
||||
|
||||
def get_access_token(jwt_token, installation_id):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['token']
|
||||
|
||||
|
||||
RunnerDescription = namedtuple('RunnerDescription', ['id', 'name', 'tags', 'offline', 'busy'])
|
||||
|
||||
def list_runners(access_token):
|
||||
headers = {
|
||||
"Authorization": f"token {access_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
|
||||
response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print("Total runners", data['total_count'])
|
||||
runners = data['runners']
|
||||
result = []
|
||||
for runner in runners:
|
||||
tags = [tag['name'] for tag in runner['labels']]
|
||||
desc = RunnerDescription(id=runner['id'], name=runner['name'], tags=tags,
|
||||
offline=runner['status']=='offline', busy=runner['busy'])
|
||||
result.append(desc)
|
||||
return result
|
||||
|
||||
def push_metrics_to_cloudwatch(listed_runners, namespace):
|
||||
import boto3
|
||||
client = boto3.client('cloudwatch')
|
||||
metrics_data = []
|
||||
busy_runners = sum(1 for runner in listed_runners if runner.busy)
|
||||
metrics_data.append({
|
||||
'MetricName': 'BusyRunners',
|
||||
'Value': busy_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
total_active_runners = sum(1 for runner in listed_runners if not runner.offline)
|
||||
metrics_data.append({
|
||||
'MetricName': 'ActiveRunners',
|
||||
'Value': total_active_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
total_runners = len(listed_runners)
|
||||
metrics_data.append({
|
||||
'MetricName': 'TotalRunners',
|
||||
'Value': total_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
if total_active_runners == 0:
|
||||
busy_ratio = 100
|
||||
else:
|
||||
busy_ratio = busy_runners / total_active_runners * 100
|
||||
|
||||
metrics_data.append({
|
||||
'MetricName': 'BusyRunnersRatio',
|
||||
'Value': busy_ratio,
|
||||
'Unit': 'Percent',
|
||||
})
|
||||
|
||||
client.put_metric_data(Namespace='RunnersMetrics', MetricData=metrics_data)
|
||||
|
||||
def main(github_secret_key, github_app_id, push_to_cloudwatch):
|
||||
payload = {
|
||||
"iat": int(time.time()) - 60,
|
||||
"exp": int(time.time()) + (10 * 60),
|
||||
"iss": github_app_id,
|
||||
}
|
||||
|
||||
encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
|
||||
installation_id = get_installation_id(encoded_jwt)
|
||||
access_token = get_access_token(encoded_jwt, installation_id)
|
||||
runners = list_runners(access_token)
|
||||
if push_to_cloudwatch:
|
||||
push_metrics_to_cloudwatch(runners, 'RunnersMetrics')
|
||||
else:
|
||||
print(runners)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Get list of runners and their states')
|
||||
parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
|
||||
parser.add_argument('-k', '--private-key', help='Private key')
|
||||
parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
|
||||
parser.add_argument('--push-to-cloudwatch', action='store_true', help='Store received token in parameter store')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.private_key_path and not args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key_path and args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key:
|
||||
private_key = args.private_key
|
||||
else:
|
||||
with open(args.private_key_path, 'r') as key_file:
|
||||
private_key = key_file.read()
|
||||
|
||||
main(private_key, args.app_id, args.push_to_cloudwatch)
|
||||
3
tests/ci/metrics_lambda/requirements.txt
Normal file
3
tests/ci/metrics_lambda/requirements.txt
Normal file
@ -0,0 +1,3 @@
|
||||
requests
|
||||
PyJWT
|
||||
cryptography
|
||||
41
tests/ci/pr_info.py
Normal file
41
tests/ci/pr_info.py
Normal file
@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
import requests
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import urllib
|
||||
from unidiff import PatchSet
|
||||
|
||||
|
||||
class PRInfo:
|
||||
def __init__(self, github_event, need_orgs=False, need_changed_files=False):
|
||||
self.number = github_event['number']
|
||||
if 'after' in github_event:
|
||||
self.sha = github_event['after']
|
||||
else:
|
||||
self.sha = github_event['pull_request']['head']['sha']
|
||||
|
||||
self.labels = set([l['name'] for l in github_event['pull_request']['labels']])
|
||||
self.user_login = github_event['pull_request']['user']['login']
|
||||
self.user_orgs = set([])
|
||||
if need_orgs:
|
||||
user_orgs_response = requests.get(github_event['pull_request']['user']['organizations_url'])
|
||||
if user_orgs_response.ok:
|
||||
response_json = user_orgs_response.json()
|
||||
self.user_orgs = set(org['id'] for org in response_json)
|
||||
|
||||
self.changed_files = set([])
|
||||
if need_changed_files:
|
||||
diff_url = github_event['pull_request']['diff_url']
|
||||
diff = urllib.request.urlopen(github_event['pull_request']['diff_url'])
|
||||
diff_object = PatchSet(diff, diff.headers.get_charsets()[0])
|
||||
self.changed_files = set([f.path for f in diff_object])
|
||||
|
||||
def get_dict(self):
|
||||
return {
|
||||
'sha': self.sha,
|
||||
'number': self.number,
|
||||
'labels': self.labels,
|
||||
'user_login': self.user_login,
|
||||
'user_orgs': self.user_orgs,
|
||||
}
|
||||
139
tests/ci/pvs_check.py
Normal file
139
tests/ci/pvs_check.py
Normal file
@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
import subprocess
|
||||
import os
|
||||
import json
|
||||
import logging
|
||||
from github import Github
|
||||
from report import create_test_html_report
|
||||
from s3_helper import S3Helper
|
||||
from pr_info import PRInfo
|
||||
import shutil
|
||||
import sys
|
||||
from get_robot_token import get_best_robot_token
|
||||
|
||||
NAME = 'PVS Studio (actions)'
|
||||
LICENCE_NAME = 'Free license: ClickHouse, Yandex'
|
||||
HTML_REPORT_FOLDER = 'pvs-studio-html-report'
|
||||
TXT_REPORT_NAME = 'pvs-studio-task-report.txt'
|
||||
|
||||
def process_logs(s3_client, additional_logs, s3_path_prefix):
|
||||
additional_urls = []
|
||||
for log_path in additional_logs:
|
||||
if log_path:
|
||||
additional_urls.append(
|
||||
s3_client.upload_test_report_to_s3(
|
||||
log_path,
|
||||
s3_path_prefix + "/" + os.path.basename(log_path)))
|
||||
|
||||
return additional_urls
|
||||
|
||||
def _process_txt_report(path):
|
||||
warnings = []
|
||||
errors = []
|
||||
with open(path, 'r') as report_file:
|
||||
for line in report_file:
|
||||
if 'viva64' in line:
|
||||
continue
|
||||
elif 'warn' in line:
|
||||
warnings.append(':'.join(line.split('\t')[0:2]))
|
||||
elif 'err' in line:
|
||||
errors.append(':'.join(line.split('\t')[0:2]))
|
||||
return warnings, errors
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
|
||||
s3_path_prefix = str(pr_number) + "/" + commit_sha + "/" + NAME.lower().replace(' ', '_')
|
||||
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
|
||||
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
|
||||
branch_name = "master"
|
||||
if pr_number != 0:
|
||||
branch_name = "PR #{}".format(pr_number)
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
|
||||
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
|
||||
|
||||
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
|
||||
raw_log_url = additional_urls[0]
|
||||
additional_urls.pop(0)
|
||||
|
||||
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
|
||||
with open('report.html', 'w') as f:
|
||||
f.write(html_report)
|
||||
|
||||
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
|
||||
logging.info("Search result in url %s", url)
|
||||
return url
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
repo_path = os.path.join(os.getenv("REPO_COPY", os.path.abspath("../../")))
|
||||
temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'pvs_check')
|
||||
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
pr_info = PRInfo(event)
|
||||
# this check modify repository so copy it to the temp directory
|
||||
logging.info("Repo copy path %s", repo_path)
|
||||
|
||||
gh = Github(get_best_robot_token())
|
||||
|
||||
images_path = os.path.join(temp_path, 'changed_images.json')
|
||||
docker_image = 'clickhouse/pvs-test'
|
||||
if os.path.exists(images_path):
|
||||
logging.info("Images file exists")
|
||||
with open(images_path, 'r') as images_fd:
|
||||
images = json.load(images_fd)
|
||||
logging.info("Got images %s", images)
|
||||
if 'clickhouse/pvs-test' in images:
|
||||
docker_image += ':' + images['clickhouse/pvs-test']
|
||||
|
||||
logging.info("Got docker image %s", docker_image)
|
||||
|
||||
s3_helper = S3Helper('https://s3.amazonaws.com')
|
||||
|
||||
licence_key = os.getenv('PVS_STUDIO_KEY')
|
||||
cmd = f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --volume={repo_path}:/repo_folder --volume={temp_path}:/test_output -e LICENCE_NAME='{LICENCE_NAME}' -e LICENCE_KEY='{licence_key}' {docker_image}"
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
|
||||
try:
|
||||
subprocess.check_output(cmd, shell=True)
|
||||
except:
|
||||
commit.create_status(context=NAME, description='PVS report failed to build', state='failure', target_url=f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}")
|
||||
sys.exit(1)
|
||||
|
||||
try:
|
||||
s3_path_prefix = str(pr_info.number) + "/" + pr_info.sha + "/" + NAME.lower().replace(' ', '_')
|
||||
html_urls = s3_helper.upload_test_folder_to_s3(os.path.join(temp_path, HTML_REPORT_FOLDER), s3_path_prefix)
|
||||
index_html = None
|
||||
|
||||
for url in html_urls:
|
||||
if 'index.html' in url:
|
||||
index_html = '<a href="{}">HTML report</a>'.format(url)
|
||||
break
|
||||
|
||||
if not index_html:
|
||||
commit.create_status(context=NAME, description='PVS report failed to build', state='failure', target_url=f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}")
|
||||
sys.exit(1)
|
||||
|
||||
txt_report = os.path.join(temp_path, TXT_REPORT_NAME)
|
||||
warnings, errors = _process_txt_report(txt_report)
|
||||
errors = errors + warnings
|
||||
|
||||
status = 'success'
|
||||
test_results = [(index_html, "Look at the report"), ("Errors count not checked", "OK")]
|
||||
description = "Total errors {}".format(len(errors))
|
||||
additional_logs = [txt_report, os.path.join(temp_path, 'pvs-studio.log')]
|
||||
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_logs)
|
||||
|
||||
print("::notice ::Report url: {}".format(report_url))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=status, target_url=report_url)
|
||||
except Exception as ex:
|
||||
print("Got an exception", ex)
|
||||
sys.exit(1)
|
||||
298
tests/ci/report.py
Normal file
298
tests/ci/report.py
Normal file
@ -0,0 +1,298 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import os
|
||||
import datetime
|
||||
|
||||
### FIXME: BEST FRONTEND PRACTICIES BELOW
|
||||
|
||||
HTML_BASE_TEST_TEMPLATE = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<style>
|
||||
@font-face {{
|
||||
font-family:'Yandex Sans Display Web';
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
|
||||
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
|
||||
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
|
||||
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
|
||||
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
|
||||
font-weight:400;
|
||||
font-style:normal;
|
||||
font-stretch:normal
|
||||
}}
|
||||
|
||||
body {{ font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }}
|
||||
h1 {{ margin-left: 10px; }}
|
||||
th, td {{ border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
|
||||
td {{ white-space: pre; font-family: Monospace, Courier New; }}
|
||||
border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
|
||||
a {{ color: #06F; text-decoration: none; }}
|
||||
a:hover, a:active {{ color: #F40; text-decoration: underline; }}
|
||||
table {{ border: 0; }}
|
||||
.main {{ margin-left: 10%; }}
|
||||
p.links a {{ padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
|
||||
th {{ cursor: pointer; }}
|
||||
|
||||
</style>
|
||||
<title>{title}</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="main">
|
||||
|
||||
<h1>{header}</h1>
|
||||
<p class="links">
|
||||
<a href="{raw_log_url}">{raw_log_name}</a>
|
||||
<a href="{commit_url}">Commit</a>
|
||||
{additional_urls}
|
||||
<a href="{task_url}">Task (github actions)</a>
|
||||
</p>
|
||||
{test_part}
|
||||
</body>
|
||||
<script type="text/javascript">
|
||||
/// Straight from https://stackoverflow.com/questions/14267781/sorting-html-table-with-javascript
|
||||
|
||||
const getCellValue = (tr, idx) => tr.children[idx].innerText || tr.children[idx].textContent;
|
||||
|
||||
const comparer = (idx, asc) => (a, b) => ((v1, v2) =>
|
||||
v1 !== '' && v2 !== '' && !isNaN(v1) && !isNaN(v2) ? v1 - v2 : v1.toString().localeCompare(v2)
|
||||
)(getCellValue(asc ? a : b, idx), getCellValue(asc ? b : a, idx));
|
||||
|
||||
// do the work...
|
||||
document.querySelectorAll('th').forEach(th => th.addEventListener('click', (() => {{
|
||||
const table = th.closest('table');
|
||||
Array.from(table.querySelectorAll('tr:nth-child(n+2)'))
|
||||
.sort(comparer(Array.from(th.parentNode.children).indexOf(th), this.asc = !this.asc))
|
||||
.forEach(tr => table.appendChild(tr) );
|
||||
}})));
|
||||
</script>
|
||||
</html>
|
||||
"""
|
||||
|
||||
HTML_TEST_PART = """
|
||||
<table>
|
||||
<tr>
|
||||
{headers}
|
||||
</tr>
|
||||
{rows}
|
||||
</table>
|
||||
"""
|
||||
|
||||
BASE_HEADERS = ['Test name', 'Test status']
|
||||
|
||||
|
||||
def _format_header(header, branch_name, branch_url=None):
|
||||
result = ' '.join([w.capitalize() for w in header.split(' ')])
|
||||
result = result.replace("Clickhouse", "ClickHouse")
|
||||
result = result.replace("clickhouse", "ClickHouse")
|
||||
if 'ClickHouse' not in result:
|
||||
result = 'ClickHouse ' + result
|
||||
result += ' for '
|
||||
if branch_url:
|
||||
result += '<a href="{url}">{name}</a>'.format(url=branch_url, name=branch_name)
|
||||
else:
|
||||
result += branch_name
|
||||
return result
|
||||
|
||||
|
||||
def _get_status_style(status):
|
||||
style = "font-weight: bold;"
|
||||
if status in ('OK', 'success', 'PASSED'):
|
||||
style += 'color: #0A0;'
|
||||
elif status in ('FAIL', 'failure', 'error', 'FAILED', 'Timeout'):
|
||||
style += 'color: #F00;'
|
||||
else:
|
||||
style += 'color: #FFB400;'
|
||||
return style
|
||||
|
||||
|
||||
def _get_html_url(url):
|
||||
if isinstance(url, str):
|
||||
return '<a href="{url}">{name}</a>'.format(url=url, name=os.path.basename(url))
|
||||
if isinstance(url, tuple):
|
||||
return '<a href="{url}">{name}</a>'.format(url=url[0], name=url[1])
|
||||
return ''
|
||||
|
||||
|
||||
def create_test_html_report(header, test_result, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls=[]):
|
||||
if test_result:
|
||||
rows_part = ""
|
||||
num_fails = 0
|
||||
has_test_time = False
|
||||
has_test_logs = False
|
||||
for result in test_result:
|
||||
test_name = result[0]
|
||||
test_status = result[1]
|
||||
|
||||
test_logs = None
|
||||
test_time = None
|
||||
if len(result) > 2:
|
||||
test_time = result[2]
|
||||
has_test_time = True
|
||||
|
||||
if len(result) > 3:
|
||||
test_logs = result[3]
|
||||
has_test_logs = True
|
||||
|
||||
row = "<tr>"
|
||||
row += "<td>" + test_name + "</td>"
|
||||
style = _get_status_style(test_status)
|
||||
|
||||
# Allow to quickly scroll to the first failure.
|
||||
is_fail = test_status == "FAIL" or test_status == 'FLAKY'
|
||||
is_fail_id = ""
|
||||
if is_fail:
|
||||
num_fails = num_fails + 1
|
||||
is_fail_id = 'id="fail' + str(num_fails) + '" '
|
||||
|
||||
row += '<td ' + is_fail_id + 'style="{}">'.format(style) + test_status + "</td>"
|
||||
|
||||
if test_time is not None:
|
||||
row += "<td>" + test_time + "</td>"
|
||||
|
||||
if test_logs is not None:
|
||||
test_logs_html = "<br>".join([_get_html_url(url) for url in test_logs])
|
||||
row += "<td>" + test_logs_html + "</td>"
|
||||
|
||||
row += "</tr>"
|
||||
rows_part += row
|
||||
|
||||
headers = BASE_HEADERS
|
||||
if has_test_time:
|
||||
headers.append('Test time, sec.')
|
||||
if has_test_logs:
|
||||
headers.append('Logs')
|
||||
|
||||
headers = ''.join(['<th>' + h + '</th>' for h in headers])
|
||||
test_part = HTML_TEST_PART.format(headers=headers, rows=rows_part)
|
||||
else:
|
||||
test_part = ""
|
||||
|
||||
additional_html_urls = ""
|
||||
for url in additional_urls:
|
||||
additional_html_urls += ' ' + _get_html_url(url)
|
||||
|
||||
result = HTML_BASE_TEST_TEMPLATE.format(
|
||||
title=_format_header(header, branch_name),
|
||||
header=_format_header(header, branch_name, branch_url),
|
||||
raw_log_name=os.path.basename(raw_log_url),
|
||||
raw_log_url=raw_log_url,
|
||||
task_url=task_url,
|
||||
test_part=test_part,
|
||||
branch_name=branch_name,
|
||||
commit_url=commit_url,
|
||||
additional_urls=additional_html_urls
|
||||
)
|
||||
return result
|
||||
|
||||
|
||||
HTML_BASE_BUILD_TEMPLATE = """
|
||||
<!DOCTYPE html>
|
||||
<html>
|
||||
<head>
|
||||
<style>
|
||||
@font-face {{
|
||||
font-family:'Yandex Sans Display Web';
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot);
|
||||
src:url(https://yastatic.net/adv-www/_/H63jN0veW07XQUIA2317lr9UIm8.eot?#iefix) format('embedded-opentype'),
|
||||
url(https://yastatic.net/adv-www/_/sUYVCPUAQE7ExrvMS7FoISoO83s.woff2) format('woff2'),
|
||||
url(https://yastatic.net/adv-www/_/v2Sve_obH3rKm6rKrtSQpf-eB7U.woff) format('woff'),
|
||||
url(https://yastatic.net/adv-www/_/PzD8hWLMunow5i3RfJ6WQJAL7aI.ttf) format('truetype'),
|
||||
url(https://yastatic.net/adv-www/_/lF_KG5g4tpQNlYIgA0e77fBSZ5s.svg#YandexSansDisplayWeb-Regular) format('svg');
|
||||
font-weight:400;
|
||||
font-style:normal;
|
||||
font-stretch:normal
|
||||
}}
|
||||
|
||||
body {{ font-family: "Yandex Sans Display Web", Arial, sans-serif; background: #EEE; }}
|
||||
h1 {{ margin-left: 10px; }}
|
||||
th, td {{ border: 0; padding: 5px 10px 5px 10px; text-align: left; vertical-align: top; line-height: 1.5; background-color: #FFF;
|
||||
border: 0; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
|
||||
a {{ color: #06F; text-decoration: none; }}
|
||||
a:hover, a:active {{ color: #F40; text-decoration: underline; }}
|
||||
table {{ border: 0; }}
|
||||
.main {{ margin: auto; }}
|
||||
p.links a {{ padding: 5px; margin: 3px; background: #FFF; line-height: 2; white-space: nowrap; box-shadow: 0 0 0 1px rgba(0, 0, 0, 0.05), 0 8px 25px -5px rgba(0, 0, 0, 0.1); }}
|
||||
tr:hover td {{filter: brightness(95%);}}
|
||||
</style>
|
||||
<title>{title}</title>
|
||||
</head>
|
||||
<body>
|
||||
<div class="main">
|
||||
<h1>{header}</h1>
|
||||
<table>
|
||||
<tr>
|
||||
<th>Compiler</th>
|
||||
<th>Build type</th>
|
||||
<th>Sanitizer</th>
|
||||
<th>Bundled</th>
|
||||
<th>Splitted</th>
|
||||
<th>Status</th>
|
||||
<th>Build log</th>
|
||||
<th>Build time</th>
|
||||
<th class="artifacts">Artifacts</th>
|
||||
</tr>
|
||||
{rows}
|
||||
</table>
|
||||
<p class="links">
|
||||
<a href="{commit_url}">Commit</a>
|
||||
<a href="{task_url}">Task (private network)</a>
|
||||
</p>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
LINK_TEMPLATE = '<a href="{url}">{text}</a>'
|
||||
|
||||
|
||||
def create_build_html_report(header, build_results, build_logs_urls, artifact_urls_list, task_url, branch_url, branch_name, commit_url):
|
||||
rows = ""
|
||||
for (build_result, build_log_url, artifact_urls) in zip(build_results, build_logs_urls, artifact_urls_list):
|
||||
row = "<tr>"
|
||||
row += "<td>{}</td>".format(build_result.compiler)
|
||||
if build_result.build_type:
|
||||
row += "<td>{}</td>".format(build_result.build_type)
|
||||
else:
|
||||
row += "<td>{}</td>".format("relwithdebuginfo")
|
||||
if build_result.sanitizer:
|
||||
row += "<td>{}</td>".format(build_result.sanitizer)
|
||||
else:
|
||||
row += "<td>{}</td>".format("none")
|
||||
|
||||
row += "<td>{}</td>".format(build_result.bundled)
|
||||
row += "<td>{}</td>".format(build_result.splitted)
|
||||
|
||||
if build_result.status:
|
||||
style = _get_status_style(build_result.status)
|
||||
row += '<td style="{}">{}</td>'.format(style, build_result.status)
|
||||
else:
|
||||
style = _get_status_style("error")
|
||||
row += '<td style="{}">{}</td>'.format(style, "error")
|
||||
|
||||
row += '<td><a href="{}">link</a></td>'.format(build_log_url)
|
||||
|
||||
if build_result.elapsed_seconds:
|
||||
delta = datetime.timedelta(seconds=build_result.elapsed_seconds)
|
||||
else:
|
||||
delta = 'unknown'
|
||||
|
||||
row += '<td>{}</td>'.format(str(delta))
|
||||
|
||||
links = ""
|
||||
link_separator = "<br/>"
|
||||
if artifact_urls:
|
||||
for artifact_url in artifact_urls:
|
||||
links += LINK_TEMPLATE.format(text=os.path.basename(artifact_url), url=artifact_url)
|
||||
links += link_separator
|
||||
if links:
|
||||
links = links[:-len(link_separator)]
|
||||
row += "<td>{}</td>".format(links)
|
||||
|
||||
row += "</tr>"
|
||||
rows += row
|
||||
return HTML_BASE_BUILD_TEMPLATE.format(
|
||||
title=_format_header(header, branch_name),
|
||||
header=_format_header(header, branch_name, branch_url),
|
||||
rows=rows,
|
||||
task_url=task_url,
|
||||
branch_name=branch_name,
|
||||
commit_url=commit_url)
|
||||
126
tests/ci/run_check.py
Normal file
126
tests/ci/run_check.py
Normal file
@ -0,0 +1,126 @@
|
||||
#!/usr/bin/env python3
|
||||
import os
|
||||
import json
|
||||
import requests
|
||||
from pr_info import PRInfo
|
||||
import sys
|
||||
import logging
|
||||
from github import Github
|
||||
from get_robot_token import get_best_robot_token
|
||||
|
||||
NAME = 'Run Check (actions)'
|
||||
|
||||
TRUSTED_ORG_IDS = {
|
||||
7409213, # yandex
|
||||
28471076, # altinity
|
||||
54801242, # clickhouse
|
||||
}
|
||||
|
||||
OK_TEST_LABEL = set(["can be tested", "release", "pr-documentation", "pr-doc-fix"])
|
||||
DO_NOT_TEST_LABEL = "do not test"
|
||||
|
||||
# Individual trusted contirbutors who are not in any trusted organization.
|
||||
# Can be changed in runtime: we will append users that we learned to be in
|
||||
# a trusted org, to save GitHub API calls.
|
||||
TRUSTED_CONTRIBUTORS = {
|
||||
"achimbab",
|
||||
"adevyatova ", # DOCSUP
|
||||
"Algunenano", # Raúl Marín, Tinybird
|
||||
"AnaUvarova", # DOCSUP
|
||||
"anauvarova", # technical writer, Yandex
|
||||
"annvsh", # technical writer, Yandex
|
||||
"atereh", # DOCSUP
|
||||
"azat",
|
||||
"bharatnc", # Newbie, but already with many contributions.
|
||||
"bobrik", # Seasoned contributor, CloundFlare
|
||||
"BohuTANG",
|
||||
"damozhaeva", # DOCSUP
|
||||
"den-crane",
|
||||
"gyuton", # DOCSUP
|
||||
"gyuton", # technical writer, Yandex
|
||||
"hagen1778", # Roman Khavronenko, seasoned contributor
|
||||
"hczhcz",
|
||||
"hexiaoting", # Seasoned contributor
|
||||
"ildus", # adjust, ex-pgpro
|
||||
"javisantana", # a Spanish ClickHouse enthusiast, ex-Carto
|
||||
"ka1bi4", # DOCSUP
|
||||
"kirillikoff", # DOCSUP
|
||||
"kitaisreal", # Seasoned contributor
|
||||
"kreuzerkrieg",
|
||||
"lehasm", # DOCSUP
|
||||
"michon470", # DOCSUP
|
||||
"MyroTk", # Tester in Altinity
|
||||
"myrrc", # Michael Kot, Altinity
|
||||
"nikvas0",
|
||||
"nvartolomei",
|
||||
"olgarev", # DOCSUP
|
||||
"otrazhenia", # Yandex docs contractor
|
||||
"pdv-ru", # DOCSUP
|
||||
"podshumok", # cmake expert from QRator Labs
|
||||
"s-mx", # Maxim Sabyanin, former employee, present contributor
|
||||
"sevirov", # technical writer, Yandex
|
||||
"spongedu", # Seasoned contributor
|
||||
"ucasFL", # Amos Bird's friend
|
||||
"vdimir", # Employee
|
||||
"vzakaznikov",
|
||||
"YiuRULE",
|
||||
"zlobober" # Developer of YT
|
||||
}
|
||||
|
||||
|
||||
def pr_is_by_trusted_user(pr_user_login, pr_user_orgs):
|
||||
if pr_user_login in TRUSTED_CONTRIBUTORS:
|
||||
logging.info("User '{}' is trusted".format(pr_user_login))
|
||||
return True
|
||||
|
||||
logging.info("User '{}' is not trusted".format(pr_user_login))
|
||||
|
||||
for org_id in pr_user_orgs:
|
||||
if org_id in TRUSTED_ORG_IDS:
|
||||
logging.info("Org '{}' is trusted; will mark user {} as trusted".format(org_id, pr_user_login))
|
||||
return True
|
||||
logging.info("Org '{}' is not trusted".format(org_id))
|
||||
|
||||
return False
|
||||
|
||||
# Returns whether we should look into individual checks for this PR. If not, it
|
||||
# can be skipped entirely.
|
||||
def should_run_checks_for_pr(pr_info):
|
||||
# Consider the labels and whether the user is trusted.
|
||||
force_labels = set(['force tests']).intersection(pr_info.labels)
|
||||
if force_labels:
|
||||
return True, "Labeled '{}'".format(', '.join(force_labels))
|
||||
|
||||
if 'do not test' in pr_info.labels:
|
||||
return False, "Labeled 'do not test'"
|
||||
|
||||
if 'can be tested' not in pr_info.labels and not pr_is_by_trusted_user(pr_info.user_login, pr_info.user_orgs):
|
||||
return False, "Needs 'can be tested' label"
|
||||
|
||||
if 'release' in pr_info.labels or 'pr-backport' in pr_info.labels or 'pr-cherrypick' in pr_info.labels:
|
||||
return False, "Don't try new checks for release/backports/cherry-picks"
|
||||
|
||||
return True, "No special conditions apply"
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
|
||||
pr_info = PRInfo(event, need_orgs=True)
|
||||
can_run, description = should_run_checks_for_pr(pr_info)
|
||||
gh = Github(get_best_robot_token())
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
if not can_run:
|
||||
print("::notice ::Cannot run")
|
||||
commit.create_status(context=NAME, description=description, state="failure", target_url=url)
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("::notice ::Can run")
|
||||
commit.create_status(context=NAME, description=description, state="pending", target_url=url)
|
||||
100
tests/ci/s3_helper.py
Normal file
100
tests/ci/s3_helper.py
Normal file
@ -0,0 +1,100 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import boto3
|
||||
from botocore.exceptions import ClientError, BotoCoreError
|
||||
from multiprocessing.dummy import Pool
|
||||
from compress_files import compress_file_fast
|
||||
from get_robot_token import get_parameter_from_ssm
|
||||
|
||||
def _md5(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
with open(fname, "rb") as f:
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hash_md5.update(chunk)
|
||||
logging.debug("MD5 for {} is {}".format(fname, hash_md5.hexdigest()))
|
||||
return hash_md5.hexdigest()
|
||||
|
||||
|
||||
def _flatten_list(lst):
|
||||
result = []
|
||||
for elem in lst:
|
||||
if isinstance(elem, list):
|
||||
result += _flatten_list(elem)
|
||||
else:
|
||||
result.append(elem)
|
||||
return result
|
||||
|
||||
|
||||
class S3Helper(object):
|
||||
def __init__(self, host):
|
||||
self.session = boto3.session.Session(region_name='us-east-1')
|
||||
self.client = self.session.client('s3', endpoint_url=host)
|
||||
|
||||
def _upload_file_to_s3(self, bucket_name, file_path, s3_path):
|
||||
logging.debug("Start uploading {} to bucket={} path={}".format(file_path, bucket_name, s3_path))
|
||||
metadata = {}
|
||||
if os.path.getsize(file_path) < 64 * 1024 * 1024:
|
||||
if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"):
|
||||
metadata['ContentType'] = "text/plain; charset=utf-8"
|
||||
logging.info("Content type %s for file path %s", "text/plain; charset=utf-8", file_path)
|
||||
elif s3_path.endswith("html"):
|
||||
metadata['ContentType'] = "text/html; charset=utf-8"
|
||||
logging.info("Content type %s for file path %s", "text/html; charset=utf-8", file_path)
|
||||
else:
|
||||
logging.info("No content type provied for %s", file_path)
|
||||
else:
|
||||
if s3_path.endswith("txt") or s3_path.endswith("log") or s3_path.endswith("err") or s3_path.endswith("out"):
|
||||
logging.info("Going to compress file log file %s to %s", file_path, file_path + ".gz")
|
||||
compress_file_fast(file_path, file_path + ".gz")
|
||||
file_path += ".gz"
|
||||
s3_path += ".gz"
|
||||
else:
|
||||
logging.info("Processing file without compression")
|
||||
logging.info("File is too large, do not provide content type")
|
||||
|
||||
self.client.upload_file(file_path, bucket_name, s3_path, ExtraArgs=metadata)
|
||||
logging.info("Upload {} to {}. Meta: {}".format(file_path, s3_path, metadata))
|
||||
return "https://s3.amazonaws.com/{bucket}/{path}".format(bucket=bucket_name, path=s3_path)
|
||||
|
||||
def upload_test_report_to_s3(self, file_path, s3_path):
|
||||
return self._upload_file_to_s3('clickhouse-test-reports', file_path, s3_path)
|
||||
|
||||
def upload_build_file_to_s3(self, file_path, s3_path):
|
||||
return self._upload_file_to_s3('clickhouse-builds', file_path, s3_path)
|
||||
|
||||
def _upload_folder_to_s3(self, folder_path, s3_folder_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks):
|
||||
logging.info("Upload folder '{}' to bucket={} of s3 folder '{}'".format(folder_path, bucket_name, s3_folder_path))
|
||||
if not os.path.exists(folder_path):
|
||||
return []
|
||||
files = os.listdir(folder_path)
|
||||
if not files:
|
||||
return []
|
||||
|
||||
p = Pool(min(len(files), 5))
|
||||
|
||||
def task(file_name):
|
||||
full_fs_path = os.path.join(folder_path, file_name)
|
||||
if keep_dirs_in_s3_path:
|
||||
full_s3_path = s3_folder_path + "/" + os.path.basename(folder_path)
|
||||
else:
|
||||
full_s3_path = s3_folder_path
|
||||
|
||||
if os.path.isdir(full_fs_path):
|
||||
return self._upload_folder_to_s3(full_fs_path, full_s3_path, bucket_name, keep_dirs_in_s3_path, upload_symlinks)
|
||||
|
||||
if os.path.islink(full_fs_path):
|
||||
if upload_symlinks:
|
||||
return self._upload_file_to_s3(bucket_name, full_fs_path, full_s3_path + "/" + file_name)
|
||||
return []
|
||||
|
||||
return self._upload_file_to_s3(bucket_name, full_fs_path, full_s3_path + "/" + file_name)
|
||||
|
||||
return sorted(_flatten_list(list(p.map(task, files))))
|
||||
|
||||
def upload_build_folder_to_s3(self, folder_path, s3_folder_path, keep_dirs_in_s3_path=True, upload_symlinks=True):
|
||||
return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-builds', keep_dirs_in_s3_path, upload_symlinks)
|
||||
|
||||
def upload_test_folder_to_s3(self, folder_path, s3_folder_path):
|
||||
return self._upload_folder_to_s3(folder_path, s3_folder_path, 'clickhouse-test-reports', True, True)
|
||||
139
tests/ci/style_check.py
Normal file
139
tests/ci/style_check.py
Normal file
@ -0,0 +1,139 @@
|
||||
#!/usr/bin/env python3
|
||||
from github import Github
|
||||
from report import create_test_html_report
|
||||
import shutil
|
||||
import logging
|
||||
import subprocess
|
||||
import os
|
||||
import csv
|
||||
from s3_helper import S3Helper
|
||||
import time
|
||||
import json
|
||||
from pr_info import PRInfo
|
||||
from get_robot_token import get_best_robot_token
|
||||
|
||||
NAME = "Style Check (actions)"
|
||||
|
||||
|
||||
def process_logs(s3_client, additional_logs, s3_path_prefix):
|
||||
additional_urls = []
|
||||
for log_path in additional_logs:
|
||||
if log_path:
|
||||
additional_urls.append(
|
||||
s3_client.upload_test_report_to_s3(
|
||||
log_path,
|
||||
s3_path_prefix + "/" + os.path.basename(log_path)))
|
||||
|
||||
return additional_urls
|
||||
|
||||
|
||||
def process_result(result_folder):
|
||||
test_results = []
|
||||
additional_files = []
|
||||
# Just upload all files from result_folder.
|
||||
# If task provides processed results, then it's responsible for content of result_folder.
|
||||
if os.path.exists(result_folder):
|
||||
test_files = [f for f in os.listdir(result_folder) if os.path.isfile(os.path.join(result_folder, f))]
|
||||
additional_files = [os.path.join(result_folder, f) for f in test_files]
|
||||
|
||||
status_path = os.path.join(result_folder, "check_status.tsv")
|
||||
logging.info("Found test_results.tsv")
|
||||
status = list(csv.reader(open(status_path, 'r'), delimiter='\t'))
|
||||
if len(status) != 1 or len(status[0]) != 2:
|
||||
return "error", "Invalid check_status.tsv", test_results, additional_files
|
||||
state, description = status[0][0], status[0][1]
|
||||
|
||||
try:
|
||||
results_path = os.path.join(result_folder, "test_results.tsv")
|
||||
test_results = list(csv.reader(open(results_path, 'r'), delimiter='\t'))
|
||||
if len(test_results) == 0:
|
||||
raise Exception("Empty results")
|
||||
|
||||
return state, description, test_results, additional_files
|
||||
except Exception:
|
||||
if state == "success":
|
||||
state, description = "error", "Failed to read test_results.tsv"
|
||||
return state, description, test_results, additional_files
|
||||
|
||||
def upload_results(s3_client, pr_number, commit_sha, test_results, additional_files):
|
||||
s3_path_prefix = f"{pr_number}/{commit_sha}/style_check"
|
||||
additional_urls = process_logs(s3_client, additional_files, s3_path_prefix)
|
||||
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/commits/master"
|
||||
branch_name = "master"
|
||||
if pr_number != 0:
|
||||
branch_name = "PR #{}".format(pr_number)
|
||||
branch_url = "https://github.com/ClickHouse/ClickHouse/pull/" + str(pr_number)
|
||||
commit_url = f"https://github.com/ClickHouse/ClickHouse/commit/{commit_sha}"
|
||||
|
||||
task_url = f"https://github.com/ClickHouse/ClickHouse/actions/runs/{os.getenv('GITHUB_RUN_ID')}"
|
||||
|
||||
raw_log_url = additional_urls[0]
|
||||
additional_urls.pop(0)
|
||||
|
||||
html_report = create_test_html_report(NAME, test_results, raw_log_url, task_url, branch_url, branch_name, commit_url, additional_urls)
|
||||
with open('report.html', 'w') as f:
|
||||
f.write(html_report)
|
||||
|
||||
url = s3_client.upload_test_report_to_s3('report.html', s3_path_prefix + ".html")
|
||||
logging.info("Search result in url %s", url)
|
||||
return url
|
||||
|
||||
|
||||
def get_commit(gh, commit_sha):
|
||||
repo = gh.get_repo(os.getenv("GITHUB_REPOSITORY", "ClickHouse/ClickHouse"))
|
||||
commit = repo.get_commit(commit_sha)
|
||||
return commit
|
||||
|
||||
def update_check_with_curl(check_id):
|
||||
cmd_template = ("curl -v --request PATCH --url https://api.github.com/repos/ClickHouse/ClickHouse/check-runs/{} "
|
||||
"--header 'authorization: Bearer {}' "
|
||||
"--header 'Accept: application/vnd.github.v3+json' "
|
||||
"--header 'content-type: application/json' "
|
||||
"-d '{{\"name\" : \"hello-world-name\"}}'")
|
||||
cmd = cmd_template.format(check_id, os.getenv("GITHUB_TOKEN"))
|
||||
subprocess.check_call(cmd, shell=True)
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(level=logging.INFO)
|
||||
repo_path = os.path.join(os.getenv("GITHUB_WORKSPACE", os.path.abspath("../../")))
|
||||
temp_path = os.path.join(os.getenv("RUNNER_TEMP", os.path.abspath("./temp")), 'style_check')
|
||||
|
||||
with open(os.getenv('GITHUB_EVENT_PATH'), 'r') as event_file:
|
||||
event = json.load(event_file)
|
||||
pr_info = PRInfo(event)
|
||||
|
||||
if not os.path.exists(temp_path):
|
||||
os.makedirs(temp_path)
|
||||
|
||||
gh = Github(get_best_robot_token())
|
||||
|
||||
images_path = os.path.join(temp_path, 'changed_images.json')
|
||||
docker_image = 'clickhouse/style-test'
|
||||
if os.path.exists(images_path):
|
||||
logging.info("Images file exists")
|
||||
with open(images_path, 'r') as images_fd:
|
||||
images = json.load(images_fd)
|
||||
logging.info("Got images %s", images)
|
||||
if 'clickhouse/style-test' in images:
|
||||
docker_image += ':' + images['clickhouse/style-test']
|
||||
|
||||
logging.info("Got docker image %s", docker_image)
|
||||
for i in range(10):
|
||||
try:
|
||||
subprocess.check_output(f"docker pull {docker_image}", shell=True)
|
||||
break
|
||||
except Exception as ex:
|
||||
time.sleep(i * 3)
|
||||
logging.info("Got execption pulling docker %s", ex)
|
||||
else:
|
||||
raise Exception(f"Cannot pull dockerhub for image {docker_image}")
|
||||
|
||||
s3_helper = S3Helper('https://s3.amazonaws.com')
|
||||
|
||||
subprocess.check_output(f"docker run -u $(id -u ${{USER}}):$(id -g ${{USER}}) --cap-add=SYS_PTRACE --volume={repo_path}:/ClickHouse --volume={temp_path}:/test_output {docker_image}", shell=True)
|
||||
state, description, test_results, additional_files = process_result(temp_path)
|
||||
report_url = upload_results(s3_helper, pr_info.number, pr_info.sha, test_results, additional_files)
|
||||
print("::notice ::Report url: {}".format(report_url))
|
||||
commit = get_commit(gh, pr_info.sha)
|
||||
commit.create_status(context=NAME, description=description, state=state, target_url=report_url)
|
||||
13
tests/ci/termination_lambda/Dockerfile
Normal file
13
tests/ci/termination_lambda/Dockerfile
Normal file
@ -0,0 +1,13 @@
|
||||
FROM public.ecr.aws/lambda/python:3.9
|
||||
|
||||
# Copy function code
|
||||
COPY app.py ${LAMBDA_TASK_ROOT}
|
||||
|
||||
# Install the function's dependencies using file requirements.txt
|
||||
# from your project folder.
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
|
||||
|
||||
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
|
||||
CMD [ "app.handler" ]
|
||||
275
tests/ci/termination_lambda/app.py
Normal file
275
tests/ci/termination_lambda/app.py
Normal file
@ -0,0 +1,275 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
import argparse
|
||||
import jwt
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
from collections import namedtuple
|
||||
|
||||
def get_key_and_app_from_aws():
|
||||
import boto3
|
||||
secret_name = "clickhouse_github_secret_key"
|
||||
session = boto3.session.Session()
|
||||
client = session.client(
|
||||
service_name='secretsmanager',
|
||||
)
|
||||
get_secret_value_response = client.get_secret_value(
|
||||
SecretId=secret_name
|
||||
)
|
||||
data = json.loads(get_secret_value_response['SecretString'])
|
||||
return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
|
||||
|
||||
def get_installation_id(jwt_token):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.get("https://api.github.com/app/installations", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data[0]['id']
|
||||
|
||||
def get_access_token(jwt_token, installation_id):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['token']
|
||||
|
||||
|
||||
RunnerDescription = namedtuple('RunnerDescription', ['id', 'name', 'tags', 'offline', 'busy'])
|
||||
|
||||
def list_runners(access_token):
|
||||
headers = {
|
||||
"Authorization": f"token {access_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
|
||||
response = requests.get("https://api.github.com/orgs/ClickHouse/actions/runners", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
print("Total runners", data['total_count'])
|
||||
runners = data['runners']
|
||||
result = []
|
||||
for runner in runners:
|
||||
tags = [tag['name'] for tag in runner['labels']]
|
||||
desc = RunnerDescription(id=runner['id'], name=runner['name'], tags=tags,
|
||||
offline=runner['status']=='offline', busy=runner['busy'])
|
||||
result.append(desc)
|
||||
return result
|
||||
|
||||
def push_metrics_to_cloudwatch(listed_runners, namespace):
|
||||
import boto3
|
||||
client = boto3.client('cloudwatch')
|
||||
metrics_data = []
|
||||
busy_runners = sum(1 for runner in listed_runners if runner.busy)
|
||||
metrics_data.append({
|
||||
'MetricName': 'BusyRunners',
|
||||
'Value': busy_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
total_active_runners = sum(1 for runner in listed_runners if not runner.offline)
|
||||
metrics_data.append({
|
||||
'MetricName': 'ActiveRunners',
|
||||
'Value': total_active_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
total_runners = len(listed_runners)
|
||||
metrics_data.append({
|
||||
'MetricName': 'TotalRunners',
|
||||
'Value': total_runners,
|
||||
'Unit': 'Count',
|
||||
})
|
||||
if total_active_runners == 0:
|
||||
busy_ratio = 100
|
||||
else:
|
||||
busy_ratio = busy_runners / total_active_runners * 100
|
||||
|
||||
metrics_data.append({
|
||||
'MetricName': 'BusyRunnersRatio',
|
||||
'Value': busy_ratio,
|
||||
'Unit': 'Percent',
|
||||
})
|
||||
|
||||
client.put_metric_data(Namespace='RunnersMetrics', MetricData=metrics_data)
|
||||
|
||||
|
||||
def how_many_instances_to_kill(event_data):
|
||||
data_array = event_data['CapacityToTerminate']
|
||||
to_kill_by_zone = {}
|
||||
for av_zone in data_array:
|
||||
zone_name = av_zone['AvailabilityZone']
|
||||
to_kill = av_zone['Capacity']
|
||||
if zone_name not in to_kill_by_zone:
|
||||
to_kill_by_zone[zone_name] = 0
|
||||
|
||||
to_kill_by_zone[zone_name] += to_kill
|
||||
return to_kill_by_zone
|
||||
|
||||
def get_candidates_to_be_killed(event_data):
|
||||
data_array = event_data['Instances']
|
||||
instances_by_zone = {}
|
||||
for instance in data_array:
|
||||
zone_name = instance['AvailabilityZone']
|
||||
instance_id = instance['InstanceId']
|
||||
if zone_name not in instances_by_zone:
|
||||
instances_by_zone[zone_name] = []
|
||||
instances_by_zone[zone_name].append(instance_id)
|
||||
|
||||
return instances_by_zone
|
||||
|
||||
def delete_runner(access_token, runner):
|
||||
headers = {
|
||||
"Authorization": f"token {access_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
|
||||
response = requests.delete(f"https://api.github.com/orgs/ClickHouse/actions/runners/{runner.id}", headers=headers)
|
||||
response.raise_for_status()
|
||||
print(f"Response code deleting {runner.name} is {response.status_code}")
|
||||
return response.status_code == 204
|
||||
|
||||
|
||||
def main(github_secret_key, github_app_id, event):
|
||||
print("Got event", json.dumps(event, sort_keys=True, indent=4))
|
||||
to_kill_by_zone = how_many_instances_to_kill(event)
|
||||
instances_by_zone = get_candidates_to_be_killed(event)
|
||||
|
||||
payload = {
|
||||
"iat": int(time.time()) - 60,
|
||||
"exp": int(time.time()) + (10 * 60),
|
||||
"iss": github_app_id,
|
||||
}
|
||||
|
||||
encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
|
||||
installation_id = get_installation_id(encoded_jwt)
|
||||
access_token = get_access_token(encoded_jwt, installation_id)
|
||||
|
||||
runners = list_runners(access_token)
|
||||
|
||||
to_delete_runners = []
|
||||
instances_to_kill = []
|
||||
for zone in to_kill_by_zone:
|
||||
num_to_kill = to_kill_by_zone[zone]
|
||||
candidates = instances_by_zone[zone]
|
||||
if num_to_kill > len(candidates):
|
||||
raise Exception(f"Required to kill {num_to_kill}, but have only {len(candidates)} candidates in AV {zone}")
|
||||
|
||||
delete_for_av = []
|
||||
for candidate in candidates:
|
||||
if candidate not in set([runner.name for runner in runners]):
|
||||
print(f"Candidate {candidate} was not in runners list, simply delete it")
|
||||
instances_to_kill.append(candidate)
|
||||
|
||||
for candidate in candidates:
|
||||
if len(delete_for_av) + len(instances_to_kill) == num_to_kill:
|
||||
break
|
||||
if candidate in instances_to_kill:
|
||||
continue
|
||||
|
||||
for runner in runners:
|
||||
if runner.name == candidate:
|
||||
if not runner.busy:
|
||||
print(f"Runner {runner.name} is not busy and can be deleted from AV {zone}")
|
||||
delete_for_av.append(runner)
|
||||
else:
|
||||
print(f"Runner {runner.name} is busy, not going to delete it")
|
||||
break
|
||||
|
||||
if len(delete_for_av) < num_to_kill:
|
||||
print(f"Checked all candidates for av {zone}, get to delete {len(delete_for_av)}, but still cannot get required {num_to_kill}")
|
||||
to_delete_runners += delete_for_av
|
||||
|
||||
print("Got instances to kill: ", ', '.join(instances_to_kill))
|
||||
print("Going to delete runners:", ', '.join([runner.name for runner in to_delete_runners]))
|
||||
for runner in to_delete_runners:
|
||||
if delete_runner(access_token, runner):
|
||||
print(f"Runner {runner.name} successfuly deleted from github")
|
||||
instances_to_kill.append(runner.name)
|
||||
else:
|
||||
print(f"Cannot delete {runner.name} from github")
|
||||
|
||||
## push metrics
|
||||
#runners = list_runners(access_token)
|
||||
#push_metrics_to_cloudwatch(runners, 'RunnersMetrics')
|
||||
|
||||
response = {
|
||||
"InstanceIDs": instances_to_kill
|
||||
}
|
||||
print(response)
|
||||
return response
|
||||
|
||||
def handler(event, context):
|
||||
private_key, app_id = get_key_and_app_from_aws()
|
||||
return main(private_key, app_id, event)
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Get list of runners and their states')
|
||||
parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
|
||||
parser.add_argument('-k', '--private-key', help='Private key')
|
||||
parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.private_key_path and not args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key_path and args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key:
|
||||
private_key = args.private_key
|
||||
else:
|
||||
with open(args.private_key_path, 'r') as key_file:
|
||||
private_key = key_file.read()
|
||||
|
||||
sample_event = {
|
||||
"AutoScalingGroupARN": "arn:aws:autoscaling:us-east-1:<account-id>:autoScalingGroup:d4738357-2d40-4038-ae7e-b00ae0227003:autoScalingGroupName/my-asg",
|
||||
"AutoScalingGroupName": "my-asg",
|
||||
"CapacityToTerminate": [
|
||||
{
|
||||
"AvailabilityZone": "us-east-1b",
|
||||
"Capacity": 1,
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
},
|
||||
{
|
||||
"AvailabilityZone": "us-east-1c",
|
||||
"Capacity": 2,
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
}
|
||||
],
|
||||
"Instances": [
|
||||
{
|
||||
"AvailabilityZone": "us-east-1b",
|
||||
"InstanceId": "i-08d0b3c1a137e02a5",
|
||||
"InstanceType": "t2.nano",
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
},
|
||||
{
|
||||
"AvailabilityZone": "us-east-1c",
|
||||
"InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal",
|
||||
"InstanceType": "t2.nano",
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
},
|
||||
{
|
||||
"AvailabilityZone": "us-east-1c",
|
||||
"InstanceId": "ip-172-31-27-227.eu-west-1.compute.internal",
|
||||
"InstanceType": "t2.nano",
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
},
|
||||
{
|
||||
"AvailabilityZone": "us-east-1c",
|
||||
"InstanceId": "ip-172-31-45-253.eu-west-1.compute.internal",
|
||||
"InstanceType": "t2.nano",
|
||||
"InstanceMarketOption": "OnDemand"
|
||||
}
|
||||
],
|
||||
"Cause": "SCALE_IN"
|
||||
}
|
||||
|
||||
main(private_key, args.app_id, sample_event)
|
||||
3
tests/ci/termination_lambda/requirements.txt
Normal file
3
tests/ci/termination_lambda/requirements.txt
Normal file
@ -0,0 +1,3 @@
|
||||
requests
|
||||
PyJWT
|
||||
cryptography
|
||||
13
tests/ci/token_lambda/Dockerfile
Normal file
13
tests/ci/token_lambda/Dockerfile
Normal file
@ -0,0 +1,13 @@
|
||||
FROM public.ecr.aws/lambda/python:3.9
|
||||
|
||||
# Copy function code
|
||||
COPY app.py ${LAMBDA_TASK_ROOT}
|
||||
|
||||
# Install the function's dependencies using file requirements.txt
|
||||
# from your project folder.
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip3 install -r requirements.txt --target "${LAMBDA_TASK_ROOT}"
|
||||
|
||||
# Set the CMD to your handler (could also be done as a parameter override outside of the Dockerfile)
|
||||
CMD [ "app.handler" ]
|
||||
106
tests/ci/token_lambda/app.py
Normal file
106
tests/ci/token_lambda/app.py
Normal file
@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import requests
|
||||
import argparse
|
||||
import jwt
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
|
||||
def get_installation_id(jwt_token):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.get("https://api.github.com/app/installations", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data[0]['id']
|
||||
|
||||
def get_access_token(jwt_token, installation_id):
|
||||
headers = {
|
||||
"Authorization": f"Bearer {jwt_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.post(f"https://api.github.com/app/installations/{installation_id}/access_tokens", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['token']
|
||||
|
||||
def get_runner_registration_token(access_token):
|
||||
headers = {
|
||||
"Authorization": f"token {access_token}",
|
||||
"Accept": "application/vnd.github.v3+json",
|
||||
}
|
||||
response = requests.post("https://api.github.com/orgs/ClickHouse/actions/runners/registration-token", headers=headers)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
return data['token']
|
||||
|
||||
def get_key_and_app_from_aws():
|
||||
import boto3
|
||||
secret_name = "clickhouse_github_secret_key"
|
||||
session = boto3.session.Session()
|
||||
client = session.client(
|
||||
service_name='secretsmanager',
|
||||
)
|
||||
get_secret_value_response = client.get_secret_value(
|
||||
SecretId=secret_name
|
||||
)
|
||||
data = json.loads(get_secret_value_response['SecretString'])
|
||||
return data['clickhouse-app-key'], int(data['clickhouse-app-id'])
|
||||
|
||||
|
||||
def main(github_secret_key, github_app_id, push_to_ssm, ssm_parameter_name):
|
||||
payload = {
|
||||
"iat": int(time.time()) - 60,
|
||||
"exp": int(time.time()) + (10 * 60),
|
||||
"iss": github_app_id,
|
||||
}
|
||||
|
||||
encoded_jwt = jwt.encode(payload, github_secret_key, algorithm="RS256")
|
||||
installation_id = get_installation_id(encoded_jwt)
|
||||
access_token = get_access_token(encoded_jwt, installation_id)
|
||||
runner_registration_token = get_runner_registration_token(access_token)
|
||||
|
||||
if push_to_ssm:
|
||||
import boto3
|
||||
|
||||
print("Trying to put params into ssm manager")
|
||||
client = boto3.client('ssm')
|
||||
client.put_parameter(
|
||||
Name=ssm_parameter_name,
|
||||
Value=runner_registration_token,
|
||||
Type='SecureString',
|
||||
Overwrite=True)
|
||||
else:
|
||||
print("Not push token to AWS Parameter Store, just print:", runner_registration_token)
|
||||
|
||||
|
||||
def handler(event, context):
|
||||
private_key, app_id = get_key_and_app_from_aws()
|
||||
main(private_key, app_id, True, 'github_runner_registration_token')
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='Get new token from github to add runners')
|
||||
parser.add_argument('-p', '--private-key-path', help='Path to file with private key')
|
||||
parser.add_argument('-k', '--private-key', help='Private key')
|
||||
parser.add_argument('-a', '--app-id', type=int, help='GitHub application ID', required=True)
|
||||
parser.add_argument('--push-to-ssm', action='store_true', help='Store received token in parameter store')
|
||||
parser.add_argument('--ssm-parameter-name', default='github_runner_registration_token', help='AWS paramater store parameter name')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.private_key_path and not args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key_path and args.private_key:
|
||||
print("Either --private-key-path or --private-key must be specified", file=sys.stderr)
|
||||
|
||||
if args.private_key:
|
||||
private_key = args.private_key
|
||||
else:
|
||||
with open(args.private_key_path, 'r') as key_file:
|
||||
private_key = key_file.read()
|
||||
|
||||
main(private_key, args.app_id, args.push_to_ssm, args.ssm_parameter_name)
|
||||
3
tests/ci/token_lambda/requirements.txt
Normal file
3
tests/ci/token_lambda/requirements.txt
Normal file
@ -0,0 +1,3 @@
|
||||
requests
|
||||
PyJWT
|
||||
cryptography
|
||||
20
tests/ci/worker/init.sh
Normal file
20
tests/ci/worker/init.sh
Normal file
@ -0,0 +1,20 @@
|
||||
#!/usr/bin/bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "Running init script"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export RUNNER_HOME=/home/ubuntu/actions-runner
|
||||
|
||||
echo "Receiving token"
|
||||
export RUNNER_TOKEN=`/usr/local/bin/aws ssm get-parameter --name github_runner_registration_token --with-decryption --output text --query Parameter.Value`
|
||||
export RUNNER_URL="https://github.com/ClickHouse"
|
||||
# Funny fact, but metadata service has fixed IP
|
||||
export INSTANCE_ID=`curl -s http://169.254.169.254/latest/meta-data/instance-id`
|
||||
|
||||
cd $RUNNER_HOME
|
||||
|
||||
echo "Going to configure runner"
|
||||
sudo -u ubuntu ./config.sh --url $RUNNER_URL --token $RUNNER_TOKEN --name $INSTANCE_ID --runnergroup Default --labels 'self-hosted,Linux,X64' --work _work
|
||||
|
||||
echo "Run"
|
||||
sudo -u ubuntu ./run.sh
|
||||
47
tests/ci/worker/ubuntu_ami.sh
Normal file
47
tests/ci/worker/ubuntu_ami.sh
Normal file
@ -0,0 +1,47 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "Running prepare script"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
export RUNNER_VERSION=2.283.1
|
||||
export RUNNER_HOME=/home/ubuntu/actions-runner
|
||||
|
||||
apt-get update
|
||||
|
||||
apt-get install --yes --no-install-recommends \
|
||||
apt-transport-https \
|
||||
ca-certificates \
|
||||
curl \
|
||||
gnupg \
|
||||
lsb-release \
|
||||
python3-pip \
|
||||
unzip
|
||||
|
||||
curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg
|
||||
|
||||
echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
|
||||
apt-get update
|
||||
|
||||
apt-get install --yes --no-install-recommends docker-ce docker-ce-cli containerd.io
|
||||
|
||||
usermod -aG docker ubuntu
|
||||
|
||||
pip install boto3 pygithub requests urllib3 unidiff
|
||||
|
||||
mkdir -p $RUNNER_HOME && cd $RUNNER_HOME
|
||||
|
||||
curl -O -L https://github.com/actions/runner/releases/download/v$RUNNER_VERSION/actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
|
||||
|
||||
tar xzf ./actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
|
||||
rm -f ./actions-runner-linux-x64-$RUNNER_VERSION.tar.gz
|
||||
./bin/installdependencies.sh
|
||||
|
||||
chown -R ubuntu:ubuntu $RUNNER_HOME
|
||||
|
||||
cd /home/ubuntu
|
||||
curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip"
|
||||
unzip awscliv2.zip
|
||||
./aws/install
|
||||
|
||||
rm -rf /home/ubuntu/awscliv2.zip /home/ubuntu/aws
|
||||
@ -985,10 +985,10 @@
|
||||
"RANGE"
|
||||
"rank"
|
||||
"rankCorr"
|
||||
"readWktMultiPolygon"
|
||||
"readWktPoint"
|
||||
"readWktPolygon"
|
||||
"readWktRing"
|
||||
"readWKTMultiPolygon"
|
||||
"readWKTPoint"
|
||||
"readWKTPolygon"
|
||||
"readWKTRing"
|
||||
"REAL"
|
||||
"REFRESH"
|
||||
"regexpQuoteMeta"
|
||||
@ -1177,6 +1177,7 @@
|
||||
"sumWithOverflow"
|
||||
"SUSPEND"
|
||||
"svg"
|
||||
"SVG"
|
||||
"SYNC"
|
||||
"synonyms"
|
||||
"SYNTAX"
|
||||
|
||||
@ -52,6 +52,7 @@
|
||||
"h3GetResolution"
|
||||
"h3EdgeLengthM"
|
||||
"svg"
|
||||
"SVG"
|
||||
"equals"
|
||||
"geohashesInBox"
|
||||
"polygonsIntersectionCartesian"
|
||||
@ -114,7 +115,7 @@
|
||||
"replaceOne"
|
||||
"emptyArrayInt32"
|
||||
"extract"
|
||||
"readWktPolygon"
|
||||
"readWKTPolygon"
|
||||
"notILike"
|
||||
"geohashDecode"
|
||||
"toModifiedJulianDay"
|
||||
@ -164,7 +165,7 @@
|
||||
"lessOrEquals"
|
||||
"subtractQuarters"
|
||||
"ngramSearch"
|
||||
"readWktRing"
|
||||
"readWKTRing"
|
||||
"trimRight"
|
||||
"endsWith"
|
||||
"ngramDistanceCaseInsensitive"
|
||||
@ -713,13 +714,13 @@
|
||||
"s2RectContains"
|
||||
"toDate"
|
||||
"regexpQuoteMeta"
|
||||
"readWktMultiPolygon"
|
||||
"readWKTMultiPolygon"
|
||||
"emptyArrayString"
|
||||
"bitmapOr"
|
||||
"cutWWW"
|
||||
"emptyArrayInt8"
|
||||
"less"
|
||||
"readWktPoint"
|
||||
"readWKTPoint"
|
||||
"reinterpretAsDateTime"
|
||||
"notEquals"
|
||||
"geoToS2"
|
||||
|
||||
@ -1,14 +1,14 @@
|
||||
SELECT readWktPoint('POINT(0 0)');
|
||||
SELECT readWktPolygon('POLYGON((1 0,10 0,10 10,0 10,1 0))');
|
||||
SELECT readWktPolygon('POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))');
|
||||
SELECT readWktMultiPolygon('MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))');
|
||||
SELECT readWKTPoint('POINT(0 0)');
|
||||
SELECT readWKTPolygon('POLYGON((1 0,10 0,10 10,0 10,1 0))');
|
||||
SELECT readWKTPolygon('POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))');
|
||||
SELECT readWKTMultiPolygon('MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))');
|
||||
|
||||
DROP TABLE IF EXISTS geo;
|
||||
CREATE TABLE geo (s String, id Int) engine=Memory();
|
||||
INSERT INTO geo VALUES ('POINT(0 0)', 1);
|
||||
INSERT INTO geo VALUES ('POINT(1 0)', 2);
|
||||
INSERT INTO geo VALUES ('POINT(2 0)', 3);
|
||||
SELECT readWktPoint(s) FROM geo ORDER BY id;
|
||||
SELECT readWKTPoint(s) FROM geo ORDER BY id;
|
||||
|
||||
DROP TABLE IF EXISTS geo;
|
||||
CREATE TABLE geo (s String, id Int) engine=Memory();
|
||||
@ -18,13 +18,13 @@ INSERT INTO geo VALUES ('POLYGON((2 0,10 0,10 10,0 10,2 0))', 3);
|
||||
INSERT INTO geo VALUES ('POLYGON((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4))', 4);
|
||||
INSERT INTO geo VALUES ('POLYGON((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4))', 5);
|
||||
INSERT INTO geo VALUES ('POLYGON((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4))', 6);
|
||||
SELECT readWktPolygon(s) FROM geo ORDER BY id;
|
||||
SELECT readWKTPolygon(s) FROM geo ORDER BY id;
|
||||
|
||||
DROP TABLE IF EXISTS geo;
|
||||
CREATE TABLE geo (s String, id Int) engine=Memory();
|
||||
INSERT INTO geo VALUES ('MULTIPOLYGON(((1 0,10 0,10 10,0 10,1 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 1);
|
||||
INSERT INTO geo VALUES ('MULTIPOLYGON(((0 0,10 0,10 10,0 10,0 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 2);
|
||||
INSERT INTO geo VALUES ('MULTIPOLYGON(((2 0,10 0,10 10,0 10,2 0),(4 4,5 4,5 5,4 5,4 4)),((-10 -10,-10 -9,-9 10,-10 -10)))', 3);
|
||||
SELECT readWktMultiPolygon(s) FROM geo ORDER BY id;
|
||||
SELECT readWKTMultiPolygon(s) FROM geo ORDER BY id;
|
||||
|
||||
DROP TABLE geo;
|
||||
|
||||
@ -1,50 +1,50 @@
|
||||
SELECT svg((0., 0.));
|
||||
SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)]);
|
||||
SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]]);
|
||||
SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]]);
|
||||
SELECT svg((0., 0.), 'b');
|
||||
SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)], 'b');
|
||||
SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], 'b');
|
||||
SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'b');
|
||||
SELECT SVG((0., 0.));
|
||||
SELECT SVG([(0., 0.), (10, 0), (10, 10), (0, 10)]);
|
||||
SELECT SVG([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]]);
|
||||
SELECT SVG([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]]);
|
||||
SELECT SVG((0., 0.), 'b');
|
||||
SELECT SVG([(0., 0.), (10, 0), (10, 10), (0, 10)], 'b');
|
||||
SELECT SVG([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], 'b');
|
||||
SELECT SVG([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'b');
|
||||
|
||||
DROP TABLE IF EXISTS geo;
|
||||
CREATE TABLE geo (p Tuple(Float64, Float64), s String, id Int) engine=Memory();
|
||||
INSERT INTO geo VALUES ((0., 0.), 'b', 1);
|
||||
INSERT INTO geo VALUES ((1., 0.), 'c', 2);
|
||||
INSERT INTO geo VALUES ((2., 0.), 'd', 3);
|
||||
SELECT svg(p) FROM geo ORDER BY id;
|
||||
SELECT svg(p, 'b') FROM geo ORDER BY id;
|
||||
SELECT svg((0., 0.), s) FROM geo ORDER BY id;
|
||||
SELECT svg(p, s) FROM geo ORDER BY id;
|
||||
SELECT SVG(p) FROM geo ORDER BY id;
|
||||
SELECT SVG(p, 'b') FROM geo ORDER BY id;
|
||||
SELECT SVG((0., 0.), s) FROM geo ORDER BY id;
|
||||
SELECT SVG(p, s) FROM geo ORDER BY id;
|
||||
|
||||
DROP TABLE IF EXISTS geo;
|
||||
CREATE TABLE geo (p Array(Tuple(Float64, Float64)), s String, id Int) engine=Memory();
|
||||
INSERT INTO geo VALUES ([(0., 0.), (10, 0), (10, 10), (0, 10)], 'b', 1);
|
||||
INSERT INTO geo VALUES ([(1., 0.), (10, 0), (10, 10), (0, 10)], 'c', 2);
|
||||
INSERT INTO geo VALUES ([(2., 0.), (10, 0), (10, 10), (0, 10)], 'd', 3);
|
||||
SELECT svg(p) FROM geo ORDER BY id;
|
||||
SELECT svg(p, 'b') FROM geo ORDER BY id;
|
||||
SELECT svg([(0., 0.), (10, 0), (10, 10), (0, 10)], s) FROM geo ORDER BY id;
|
||||
SELECT svg(p, s) FROM geo ORDER BY id;
|
||||
SELECT SVG(p) FROM geo ORDER BY id;
|
||||
SELECT SVG(p, 'b') FROM geo ORDER BY id;
|
||||
SELECT SVG([(0., 0.), (10, 0), (10, 10), (0, 10)], s) FROM geo ORDER BY id;
|
||||
SELECT SVG(p, s) FROM geo ORDER BY id;
|
||||
|
||||
DROP TABLE IF EXISTS geo;
|
||||
CREATE TABLE geo (p Array(Array(Tuple(Float64, Float64))), s String, id Int) engine=Memory();
|
||||
INSERT INTO geo VALUES ([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'b', 1);
|
||||
INSERT INTO geo VALUES ([[(1., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'c', 2);
|
||||
INSERT INTO geo VALUES ([[(2., 0.), (10, 0), (10, 10), (0, 10)], [(4, 4), (5, 4), (5, 5), (4, 5)]], 'd', 3);
|
||||
SELECT svg(p) FROM geo ORDER BY id;
|
||||
SELECT svg(p, 'b') FROM geo ORDER BY id;
|
||||
SELECT svg([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], s) FROM geo ORDER BY id;
|
||||
SELECT svg(p, s) FROM geo ORDER BY id;
|
||||
SELECT SVG(p) FROM geo ORDER BY id;
|
||||
SELECT SVG(p, 'b') FROM geo ORDER BY id;
|
||||
SELECT SVG([[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], s) FROM geo ORDER BY id;
|
||||
SELECT SVG(p, s) FROM geo ORDER BY id;
|
||||
|
||||
DROP TABLE IF EXISTS geo;
|
||||
CREATE TABLE geo (p Array(Array(Array(Tuple(Float64, Float64)))), s String, id Int) engine=Memory();
|
||||
INSERT INTO geo VALUES ([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'b', 1);
|
||||
INSERT INTO geo VALUES ([[[(1., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'c', 2);
|
||||
INSERT INTO geo VALUES ([[[(2., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], 'd', 3);
|
||||
SELECT svg(p) FROM geo ORDER BY id;
|
||||
SELECT svg(p, 'b') FROM geo ORDER BY id;
|
||||
SELECT svg([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], s) FROM geo ORDER BY id;
|
||||
SELECT svg(p, s) FROM geo ORDER BY id;
|
||||
SELECT SVG(p) FROM geo ORDER BY id;
|
||||
SELECT SVG(p, 'b') FROM geo ORDER BY id;
|
||||
SELECT SVG([[[(0., 0.), (10, 0), (10, 10), (0, 10)], [(4., 4.), (5, 4), (5, 5), (4, 5)]], [[(-10., -10.), (-10, -9), (-9, 10)]]], s) FROM geo ORDER BY id;
|
||||
SELECT SVG(p, s) FROM geo ORDER BY id;
|
||||
|
||||
DROP TABLE geo;
|
||||
|
||||
53
tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect
Executable file
53
tests/queries/0_stateless/02049_clickhouse_local_merge_tree.expect
Executable file
@ -0,0 +1,53 @@
|
||||
#!/usr/bin/expect -f
|
||||
# Tags: no-fasttest
|
||||
|
||||
log_user 0
|
||||
set timeout 20
|
||||
match_max 100000
|
||||
|
||||
# A default timeout action is to fail
|
||||
expect_after {
|
||||
timeout {
|
||||
exit 1
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
set basedir [file dirname $argv0]
|
||||
spawn bash -c "source $basedir/../shell_config.sh ; \$CLICKHOUSE_LOCAL --disable_suggestion"
|
||||
expect ":) "
|
||||
|
||||
send -- "drop table if exists t\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "create table t engine=MergeTree() order by tuple() as select 1\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "set optimize_on_insert = 0\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "drop table if exists tt\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "create table tt (date Date, version UInt64, val UInt64) engine = ReplacingMergeTree(version) partition by date order by date\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "insert into tt values ('2020-01-01', 2, 2), ('2020-01-01', 1, 1)\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "insert into tt values ('2020-01-01', 0, 0)\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "OPTIMIZE TABLE tt\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "select * from tt order by version format TSV\r"
|
||||
expect "2020-01-01\t2\t2"
|
||||
|
||||
send -- "drop table tt\r"
|
||||
expect "Ok."
|
||||
send -- "drop table t\r"
|
||||
expect "Ok."
|
||||
|
||||
send -- "\4"
|
||||
expect eof
|
||||
@ -0,0 +1,4 @@
|
||||
0
|
||||
SelectedRows: 131010 (increment)
|
||||
OK
|
||||
OK
|
||||
15
tests/queries/0_stateless/02050_client_profile_events.sh
Executable file
15
tests/queries/0_stateless/02050_client_profile_events.sh
Executable file
@ -0,0 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: long
|
||||
|
||||
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CURDIR"/../shell_config.sh
|
||||
|
||||
# do not print any ProfileEvents packets
|
||||
$CLICKHOUSE_CLIENT -q 'select * from numbers(1e5) format Null' |& grep -c 'SelectedRows'
|
||||
# print only last
|
||||
$CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=-1 -q 'select * from numbers(1e5) format Null' |& grep -o 'SelectedRows: .*$'
|
||||
# print everything
|
||||
test "$($CLICKHOUSE_CLIENT --print-profile-events -q 'select * from numbers(1e9) format Null' |& grep -c 'SelectedRows')" -gt 1 && echo OK || echo FAIL
|
||||
# print each 100 ms
|
||||
test "$($CLICKHOUSE_CLIENT --print-profile-events --profile-events-delay-ms=100 -q 'select * from numbers(1e9) format Null' |& grep -c 'SelectedRows')" -gt 1 && echo OK || echo FAIL
|
||||
@ -0,0 +1 @@
|
||||
OK
|
||||
32
tests/queries/0_stateless/02051_symlinks_to_user_files.sh
Executable file
32
tests/queries/0_stateless/02051_symlinks_to_user_files.sh
Executable file
@ -0,0 +1,32 @@
|
||||
#!/usr/bin/env bash
|
||||
# Tags: no-fasttest, no-parallel
|
||||
|
||||
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
|
||||
# shellcheck source=../shell_config.sh
|
||||
. "$CUR_DIR"/../shell_config.sh
|
||||
|
||||
# See 01658_read_file_to_string_column.sh
|
||||
user_files_path=$(clickhouse-client --query "select _path,_file from file('nonexist.txt', 'CSV', 'val1 char')" 2>&1 | grep Exception | awk '{gsub("/nonexist.txt","",$9); print $9}')
|
||||
|
||||
FILE_PATH="${user_files_path}/file/"
|
||||
mkdir -p ${FILE_PATH}
|
||||
chmod 777 ${FILE_PATH}
|
||||
|
||||
FILE="test_symlink_${CLICKHOUSE_DATABASE}"
|
||||
|
||||
symlink_path=${FILE_PATH}/${FILE}
|
||||
file_path=$CUR_DIR/${FILE}
|
||||
|
||||
touch ${file_path}
|
||||
ln -s ${file_path} ${symlink_path}
|
||||
chmod ugo+w ${symlink_path}
|
||||
|
||||
function cleanup()
|
||||
{
|
||||
rm ${symlink_path} ${file_path}
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
${CLICKHOUSE_CLIENT} --query="insert into table function file('${symlink_path}', 'Values', 'a String') select 'OK'";
|
||||
${CLICKHOUSE_CLIENT} --query="select * from file('${symlink_path}', 'Values', 'a String')";
|
||||
|
||||
@ -0,0 +1,8 @@
|
||||
1
|
||||
1
|
||||
10
|
||||
10
|
||||
100
|
||||
100
|
||||
10000
|
||||
10000
|
||||
@ -0,0 +1,19 @@
|
||||
-- Tags: long
|
||||
|
||||
{% for rows_in_table in [1, 10, 100, 10000] %}
|
||||
{% for wide in [0, 100000000] %}
|
||||
drop table if exists data_02052_{{ rows_in_table }}_wide{{ wide }};
|
||||
create table data_02052_{{ rows_in_table }}_wide{{ wide }} (key Int, value String)
|
||||
engine=MergeTree()
|
||||
order by key
|
||||
settings
|
||||
min_bytes_for_wide_part={{ wide }}
|
||||
as select number, repeat(toString(number), 5) from numbers({{ rows_in_table }});
|
||||
|
||||
-- avoid any optimizations with ignore(*)
|
||||
select count(ignore(*)) from data_02052_{{ rows_in_table }}_wide{{ wide }} settings max_read_buffer_size=1, max_threads=1;
|
||||
select count(ignore(*)) from data_02052_{{ rows_in_table }}_wide{{ wide }} settings max_read_buffer_size=0, max_threads=1; -- { serverError CANNOT_READ_ALL_DATA }
|
||||
|
||||
drop table data_02052_{{ rows_in_table }}_wide{{ wide }};
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
@ -0,0 +1,11 @@
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
1
|
||||
14
tests/queries/0_stateless/02100_replaceRegexpAll_bug.sql
Normal file
14
tests/queries/0_stateless/02100_replaceRegexpAll_bug.sql
Normal file
@ -0,0 +1,14 @@
|
||||
SELECT 'aaaabb ' == trim(leading 'b ' FROM 'b aaaabb ') x;
|
||||
SELECT 'b aaaa' == trim(trailing 'b ' FROM 'b aaaabb ') x;
|
||||
SELECT 'aaaa' == trim(both 'b ' FROM 'b aaaabb ') x;
|
||||
|
||||
SELECT '1' == replaceRegexpAll(',,1,,', '^[,]*|[,]*$', '') x;
|
||||
SELECT '1' == replaceRegexpAll(',,1', '^[,]*|[,]*$', '') x;
|
||||
SELECT '1' == replaceRegexpAll('1,,', '^[,]*|[,]*$', '') x;
|
||||
|
||||
SELECT '1,,' == replaceRegexpOne(',,1,,', '^[,]*|[,]*$', '') x;
|
||||
SELECT '1' == replaceRegexpOne(',,1', '^[,]*|[,]*$', '') x;
|
||||
SELECT '1,,' == replaceRegexpOne('1,,', '^[,]*|[,]*$', '') x;
|
||||
|
||||
SELECT '5935,5998,6014' == trim(BOTH ', ' FROM '5935,5998,6014, ') x;
|
||||
SELECT '5935,5998,6014' == replaceRegexpAll('5935,5998,6014, ', concat('^[', regexpQuoteMeta(', '), ']*|[', regexpQuoteMeta(', '), ']*$'), '') AS x;
|
||||
@ -70,7 +70,7 @@ find $ROOT_PATH/{src,base,programs,utils} -name '*.xml' |
|
||||
xargs xmllint --noout --nonet
|
||||
|
||||
# FIXME: for now only clickhouse-test
|
||||
pylint --rcfile=$ROOT_PATH/.pylintrc --score=n $ROOT_PATH/tests/clickhouse-test
|
||||
pylint --rcfile=$ROOT_PATH/.pylintrc --persistent=no --score=n $ROOT_PATH/tests/clickhouse-test
|
||||
|
||||
find $ROOT_PATH -not -path $ROOT_PATH'/contrib*' \( -name '*.yaml' -or -name '*.yml' \) -type f |
|
||||
grep -vP $EXCLUDE_DIRS |
|
||||
@ -162,7 +162,7 @@ find $ROOT_PATH -name '.gitmodules' | while read i; do grep -F 'url = ' $i | gre
|
||||
find $ROOT_PATH/{src,base,programs} -name '*.h' -or -name '*.cpp' 2>/dev/null | xargs grep -i -F 'General Public License' && echo "There shouldn't be any code snippets under GPL or LGPL"
|
||||
|
||||
# There shouldn't be any docker containers outside docker directory
|
||||
find $ROOT_PATH -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:"
|
||||
find $ROOT_PATH -not -path $ROOT_PATH'/tests/ci*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name Dockerfile -type f 2>/dev/null | xargs --no-run-if-empty -n1 echo "Please move Dockerfile to docker directory:"
|
||||
|
||||
# There shouldn't be any docker compose files outside docker directory
|
||||
#find $ROOT_PATH -not -path $ROOT_PATH'/tests/testflows*' -not -path $ROOT_PATH'/docker*' -not -path $ROOT_PATH'/contrib*' -name '*compose*.yml' -type f 2>/dev/null | xargs --no-run-if-empty grep -l "version:" | xargs --no-run-if-empty -n1 echo "Please move docker compose to docker directory:"
|
||||
|
||||
Loading…
Reference in New Issue
Block a user