thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-12-11 17:02:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into implement-23210

Browse Source
This commit is contained in:
Nikolai Kochetov 2024-11-20 13:17:10 +03:00 committed by GitHub
commit bc35be4747
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2603 changed files with 355038 additions and 6861 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
.github/workflows/merge_queue.yml vendored
View File

@ -58,13 +58,8 @@ jobs:
test_name: Style check
runner_type: style-checker-aarch64
run_command: |
python3 style_check.py
python3 style_check.py --no-push
data: ${{ needs.RunConfig.outputs.data }}
secrets:
secret_envs: |
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
RCSK
FastTest:
needs: [RunConfig, BuildDockers]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}

34
.github/workflows/nightly.yml vendored
View File

@ -27,7 +27,7 @@ jobs:
id: runconfig
run: |
echo "::group::configure CI run"
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --skip-jobs --outfile ${{ runner.temp }}/ci_run_data.json
python3 "$GITHUB_WORKSPACE/tests/ci/ci.py" --configure --workflow NightlyBuilds --outfile ${{ runner.temp }}/ci_run_data.json
echo "::endgroup::"
echo "::group::CI run configure results"
@ -44,9 +44,39 @@ jobs:
with:
data: "${{ needs.RunConfig.outputs.data }}"
set_latest: true
Builds_1:
needs: [RunConfig]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_1') }}
uses: ./.github/workflows/reusable_build_stage.yml
with:
stage: Builds_1
data: ${{ needs.RunConfig.outputs.data }}
Tests_1:
needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_1') }}
uses: ./.github/workflows/reusable_test_stage.yml
with:
stage: Tests_1
data: ${{ needs.RunConfig.outputs.data }}
Builds_2:
needs: [RunConfig, Builds_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Builds_2') }}
uses: ./.github/workflows/reusable_build_stage.yml
with:
stage: Builds_2
data: ${{ needs.RunConfig.outputs.data }}
Tests_2:
needs: [RunConfig, Builds_1, Tests_1]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).stages_data.stages_to_do, 'Tests_2') }}
uses: ./.github/workflows/reusable_test_stage.yml
with:
stage: Tests_2
data: ${{ needs.RunConfig.outputs.data }}
CheckWorkflow:
if: ${{ !cancelled() }}
needs: [RunConfig, BuildDockers]
needs: [RunConfig, BuildDockers, Tests_2]
runs-on: [self-hosted, style-checker-aarch64]
steps:
- name: Check out repository code

5
.github/workflows/pull_request.yml vendored
View File

@ -79,10 +79,7 @@ jobs:
python3 style_check.py
data: ${{ needs.RunConfig.outputs.data }}
secrets:
secret_envs: |
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
RCSK
robot_git_token: ${{secrets.ROBOT_CLICKHOUSE_SSH_KEY}}
FastTest:
needs: [RunConfig, BuildDockers, StyleCheck]
if: ${{ !failure() && !cancelled() && contains(fromJson(needs.RunConfig.outputs.data).jobs_data.jobs_to_do, 'Fast test') }}

17
.github/workflows/reusable_build.yml vendored
View File

@ -34,8 +34,11 @@ name: Build ClickHouse
description: additional ENV variables to setup the job
type: string
secrets:
secret_envs:
description: if given, it's passed to the environments
robot_git_token:
required: false
ci_db_url:
required: false
ci_db_password:
required: false
jobs:
@ -58,10 +61,18 @@ jobs:
run: |
cat >> "$GITHUB_ENV" << 'EOF'
${{inputs.additional_envs}}
${{secrets.secret_envs}}
DOCKER_TAG<<DOCKER_JSON
${{ toJson(fromJson(inputs.data).docker_data.images) }}
DOCKER_JSON
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
${{secrets.robot_git_token}}
RCSK
CI_DB_URL<<CIDBU
${{ secrets.ci_db_url }}
CIDBU
CI_DB_PASSWORD<<CIDBP
${{ secrets.ci_db_password }}
CIDBP
EOF
python3 "$GITHUB_WORKSPACE"/tests/ci/ci_config.py --build-name "${{inputs.build_name}}" >> "$GITHUB_ENV"
- name: Apply sparse checkout for contrib # in order to check that it doesn't break build

11
.github/workflows/reusable_build_stage.yml vendored
View File

@ -18,8 +18,11 @@ name: BuildStageWF
type: string
required: true
secrets:
secret_envs:
description: if given, it's passed to the environments
robot_git_token:
required: false
ci_db_url:
required: false
ci_db_password:
required: false
jobs:
@ -39,4 +42,6 @@ jobs:
checkout_depth: 0
data: ${{ inputs.data }}
secrets:
secret_envs: ${{ secrets.secret_envs }}
robot_git_token: ${{ secrets.robot_git_token }}
ci_db_url: ${{ secrets.ci_db_url }}
ci_db_password: ${{ secrets.ci_db_password }}

17
.github/workflows/reusable_simple_job.yml vendored
View File

@ -45,8 +45,11 @@ name: Simple job
type: boolean
default: false
secrets:
secret_envs:
description: if given, it's passed to the environments
robot_git_token:
required: false
ci_db_url:
required: false
ci_db_password:
required: false
@ -77,7 +80,15 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
CHECK_NAME=${{ inputs.test_name }}
${{inputs.additional_envs}}
${{secrets.secret_envs}}
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
${{secrets.robot_git_token}}
RCSK
CI_DB_URL<<CIDBU
${{ secrets.ci_db_url }}
CIDBU
CI_DB_PASSWORD<<CIDBP
${{ secrets.ci_db_password }}
CIDBP
EOF
- name: Common setup
uses: ./.github/actions/common_setup

17
.github/workflows/reusable_test.yml vendored
View File

@ -40,8 +40,11 @@ name: Testing workflow
type: string
default: "$GITHUB_WORKSPACE/tests/ci"
secrets:
secret_envs:
description: if given, it's passed to the environments
robot_git_token:
required: false
ci_db_url:
required: false
ci_db_password:
required: false
@ -75,10 +78,18 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
CHECK_NAME=${{ inputs.test_name }}
${{inputs.additional_envs}}
${{secrets.secret_envs}}
DOCKER_TAG<<DOCKER_JSON
${{ toJson(fromJson(inputs.data).docker_data.images) }}
DOCKER_JSON
ROBOT_CLICKHOUSE_SSH_KEY<<RCSK
${{secrets.robot_git_token}}
RCSK
CI_DB_URL<<CIDBU
${{ secrets.ci_db_url }}
CIDBU
CI_DB_PASSWORD<<CIDBP
${{ secrets.ci_db_password }}
CIDBP
EOF
- name: Common setup
uses: ./.github/actions/common_setup

11
.github/workflows/reusable_test_stage.yml vendored
View File

@ -15,8 +15,11 @@ name: StageWF
type: string
required: true
secrets:
secret_envs:
description: if given, it's passed to the environments
robot_git_token:
required: false
ci_db_url:
required: false
ci_db_password:
required: false
jobs:
@ -32,4 +35,6 @@ jobs:
runner_type: ${{ matrix.job_name_and_runner_type.runner_type }}
data: ${{ inputs.data }}
secrets:
secret_envs: ${{ secrets.secret_envs }}
robot_git_token: ${{ secrets.robot_git_token }}
ci_db_url: ${{ secrets.ci_db_url }}
ci_db_password: ${{ secrets.ci_db_password }}

3
.gitmodules vendored
View File

@ -1,6 +1,9 @@
# Please do not use 'branch = ...' tags with submodule entries. Such tags make updating submodules a
# little bit more convenient but they do *not* specify the tracked submodule branch. Thus, they are
# more confusing than useful.
[submodule "contrib/jwt-cpp"]
path = contrib/jwt-cpp
url = https://github.com/Thalhammer/jwt-cpp
[submodule "contrib/zstd"]
path = contrib/zstd
url = https://github.com/facebook/zstd

4
README.md
View File

@ -27,6 +27,7 @@ curl https://clickhouse.com/ | sh
* [YouTube channel](https://www.youtube.com/c/ClickHouseDB) has a lot of content about ClickHouse in video format.
* [Slack](https://clickhouse.com/slack) and [Telegram](https://telegram.me/clickhouse_en) allow chatting with ClickHouse users in real-time.
* [Blog](https://clickhouse.com/blog/) contains various ClickHouse-related articles, as well as announcements and reports about events.
* [Bluesky](https://bsky.app/profile/clickhouse.com) and [X](https://x.com/ClickHouseDB) for short news.
* [Code Browser (github.dev)](https://github.dev/ClickHouse/ClickHouse) with syntax highlighting, powered by github.dev.
* [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
@ -43,12 +44,13 @@ Keep an eye out for upcoming meetups and events around the world. Somewhere else
Upcoming meetups
* [Ghent Meetup](https://www.meetup.com/clickhouse-belgium-user-group/events/303049405/) - November 19
* [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - November 21
* [Paris Meetup](https://www.meetup.com/clickhouse-france-user-group/events/303096434) - November 26
* [Amsterdam Meetup](https://www.meetup.com/clickhouse-netherlands-user-group/events/303638814) - December 3
* [Stockholm Meetup](https://www.meetup.com/clickhouse-stockholm-user-group/events/304382411) - December 9
* [New York Meetup](https://www.meetup.com/clickhouse-new-york-user-group/events/304268174) - December 9
* [Kuala Lampur Meetup](https://www.meetup.com/clickhouse-malaysia-meetup-group/events/304576472/) - December 11
* [San Francisco Meetup](https://www.meetup.com/clickhouse-silicon-valley-meetup-group/events/304286951/) - December 12
* [Dubai Meetup](https://www.meetup.com/clickhouse-dubai-meetup-group/events/303096989/) - Feb 3
Recently completed meetups

10
base/poco/Net/include/Poco/Net/HTTPServerParams.h
View File

@ -43,7 +43,7 @@ namespace Net
/// Sets the following default values:
/// - timeout: 60 seconds
/// - keepAlive: true
/// - maxKeepAliveRequests: 0
/// - maxKeepAliveRequests: 100
/// - keepAliveTimeout: 15 seconds
void setServerName(const std::string & serverName);
@ -87,12 +87,12 @@ namespace Net
const Poco::Timespan & getKeepAliveTimeout() const;
/// Returns the connection timeout for HTTP connections.
void setMaxKeepAliveRequests(int maxKeepAliveRequests);
void setMaxKeepAliveRequests(size_t maxKeepAliveRequests);
/// Specifies the maximum number of requests allowed
/// during a persistent connection. 0 means unlimited
/// connections.
int getMaxKeepAliveRequests() const;
size_t getMaxKeepAliveRequests() const;
/// Returns the maximum number of requests allowed
/// during a persistent connection, or 0 if
/// unlimited connections are allowed.
@ -106,7 +106,7 @@ namespace Net
std::string _softwareVersion;
Poco::Timespan _timeout;
bool _keepAlive;
int _maxKeepAliveRequests;
size_t _maxKeepAliveRequests;
Poco::Timespan _keepAliveTimeout;
};
@ -138,7 +138,7 @@ namespace Net
}
inline int HTTPServerParams::getMaxKeepAliveRequests() const
inline size_t HTTPServerParams::getMaxKeepAliveRequests() const
{
return _maxKeepAliveRequests;
}

4
base/poco/Net/include/Poco/Net/HTTPServerSession.h
View File

@ -65,7 +65,7 @@ namespace Net
private:
bool _firstRequest;
Poco::Timespan _keepAliveTimeout;
int _maxKeepAliveRequests;
size_t _maxKeepAliveRequests;
};
@ -74,7 +74,7 @@ namespace Net
//
inline bool HTTPServerSession::canKeepAlive() const
{
return _maxKeepAliveRequests != 0;
return getKeepAlive() && _maxKeepAliveRequests > 0;
}

4
base/poco/Net/src/HTTPServerParams.cpp
View File

@ -22,7 +22,7 @@ namespace Net {
HTTPServerParams::HTTPServerParams():
_timeout(60000000),
_keepAlive(true),
_maxKeepAliveRequests(0),
_maxKeepAliveRequests(100),
_keepAliveTimeout(15000000)
{
}
@ -63,7 +63,7 @@ void HTTPServerParams::setKeepAliveTimeout(const Poco::Timespan& timeout)
}
void HTTPServerParams::setMaxKeepAliveRequests(int maxKeepAliveRequests)
void HTTPServerParams::setMaxKeepAliveRequests(size_t maxKeepAliveRequests)
{
poco_assert (maxKeepAliveRequests >= 0);
_maxKeepAliveRequests = maxKeepAliveRequests;

2
base/poco/Net/src/HTTPServerSession.cpp
View File

@ -50,7 +50,7 @@ bool HTTPServerSession::hasMoreRequests()
--_maxKeepAliveRequests;
return socket().poll(getTimeout(), Socket::SELECT_READ);
}
else if (_maxKeepAliveRequests != 0 && getKeepAlive())
else if (canKeepAlive())
{
if (_maxKeepAliveRequests > 0)
--_maxKeepAliveRequests;

5
base/poco/Net/src/TCPServerConnection.cpp
View File

@ -18,7 +18,6 @@
using Poco::Exception;
using Poco::ErrorHandler;
namespace Poco {
@ -31,9 +30,7 @@ TCPServerConnection::TCPServerConnection(const StreamSocket& socket):
}
TCPServerConnection::~TCPServerConnection()
{
}
TCPServerConnection::~TCPServerConnection() = default;
void TCPServerConnection::start()

14
ci/docker/stateful-test/Dockerfile
View File

@ -1,14 +0,0 @@
ARG FROM_TAG=latest
FROM clickhouse/stateless-test:$FROM_TAG
USER root
RUN apt-get update -y \
&& env DEBIAN_FRONTEND=noninteractive \
apt-get install --yes --no-install-recommends \
nodejs \
npm \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* \
USER clickhouse

117
ci/docker/stateless-test/Dockerfile
View File

@ -1,117 +0,0 @@
# docker build -t clickhouse/stateless-test .
FROM ubuntu:22.04
# ARG for quick switch to a given ubuntu mirror
ARG apt_archive="http://archive.ubuntu.com"
RUN sed -i "s|http://archive.ubuntu.com|$apt_archive|g" /etc/apt/sources.list
ARG odbc_driver_url="https://github.com/ClickHouse/clickhouse-odbc/releases/download/v1.1.6.20200320/clickhouse-odbc-1.1.6-Linux.tar.gz"
RUN mkdir /etc/clickhouse-server /etc/clickhouse-keeper /etc/clickhouse-client && chmod 777 /etc/clickhouse-* \
&& mkdir -p /var/lib/clickhouse /var/log/clickhouse-server && chmod 777 /var/log/clickhouse-server /var/lib/clickhouse
RUN addgroup --gid 1001 clickhouse && adduser --uid 1001 --gid 1001 --disabled-password clickhouse
# moreutils - provides ts fo FT
# expect, bzip2 - requried by FT
# bsdmainutils - provides hexdump for FT
# golang version 1.13 on Ubuntu 20 is enough for tests
RUN apt-get update -y \
&& env DEBIAN_FRONTEND=noninteractive \
apt-get install --yes --no-install-recommends \
awscli \
brotli \
lz4 \
expect \
moreutils \
bzip2 \
bsdmainutils \
golang \
lsof \
mysql-client=8.0* \
ncdu \
netcat-openbsd \
nodejs \
npm \
odbcinst \
openjdk-11-jre-headless \
openssl \
postgresql-client \
python3 \
python3-pip \
qemu-user-static \
sqlite3 \
sudo \
tree \
unixodbc \
rustc \
cargo \
zstd \
file \
jq \
pv \
zip \
unzip \
p7zip-full \
curl \
wget \
xz-utils \
&& apt-get clean \
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
ARG PROTOC_VERSION=25.1
RUN curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip \
&& unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local \
&& rm protoc-${PROTOC_VERSION}-linux-x86_64.zip
COPY requirements.txt /
RUN pip3 install --no-cache-dir -r /requirements.txt
RUN mkdir -p /tmp/clickhouse-odbc-tmp \
&& cd /tmp/clickhouse-odbc-tmp \
&& curl -L ${odbc_driver_url} | tar --strip-components=1 -xz clickhouse-odbc-1.1.6-Linux \
&& mkdir /usr/local/lib64 -p \
&& cp /tmp/clickhouse-odbc-tmp/lib64/*.so /usr/local/lib64/ \
&& odbcinst -i -d -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbcinst.ini.sample \
&& odbcinst -i -s -l -f /tmp/clickhouse-odbc-tmp/share/doc/clickhouse-odbc/config/odbc.ini.sample \
&& sed -i 's"=libclickhouseodbc"=/usr/local/lib64/libclickhouseodbc"' /etc/odbcinst.ini \
&& rm -rf /tmp/clickhouse-odbc-tmp
ENV TZ=Europe/Amsterdam
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
ENV NUM_TRIES=1
# Unrelated to vars in setup_minio.sh, but should be the same there
# to have the same binaries for local running scenario
ARG MINIO_SERVER_VERSION=2024-08-03T04-33-23Z
ARG MINIO_CLIENT_VERSION=2024-07-31T15-58-33Z
ARG TARGETARCH
# Download Minio-related binaries
RUN arch=${TARGETARCH:-amd64} \
&& curl -L "https://dl.min.io/server/minio/release/linux-${arch}/archive/minio.RELEASE.${MINIO_SERVER_VERSION}" -o /minio \
&& curl -L "https://dl.min.io/client/mc/release/linux-${arch}/archive/mc.RELEASE.${MINIO_CLIENT_VERSION}" -o /mc \
&& chmod +x /mc /minio
ENV MINIO_ROOT_USER="clickhouse"
ENV MINIO_ROOT_PASSWORD="clickhouse"
# for minio to work without root
RUN chmod 777 /home
ENV HOME="/home"
ENV TEMP_DIR="/tmp/praktika"
ENV PATH="/wd/tests:/tmp/praktika/input:$PATH"
RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoop-3.3.1/hadoop-3.3.1.tar.gz' \
&& tar -xvf hadoop-3.3.1.tar.gz \
&& rm -rf hadoop-3.3.1.tar.gz \
&& chmod 777 /hadoop-3.3.1
RUN npm install -g azurite@3.30.0 \
&& npm install -g tslib && npm install -g node
USER clickhouse

6
ci/docker/stateless-test/requirements.txt
View File

@ -1,6 +0,0 @@
Jinja2==3.1.3
numpy==1.26.4
requests==2.32.3
pandas==1.5.3
scipy==1.12.0
pyarrow==18.0.0

74
ci/jobs/build_clickhouse.py
View File

@ -13,30 +13,11 @@ class JobStages(metaclass=MetaClasses.WithIter):
def parse_args():
parser = argparse.ArgumentParser(description="ClickHouse Build Job")
parser.add_argument(
"--build-type",
help="Type: <amd|arm>,<debug|release>,<asan|msan|..>",
)
parser.add_argument(
"--param",
help="Optional user-defined job start stage (for local run)",
default=None,
)
parser.add_argument("BUILD_TYPE", help="Type: <amd|arm_debug|release_sanitizer>")
parser.add_argument("--param", help="Optional custom job start stage", default=None)
return parser.parse_args()
CMAKE_CMD = """cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA \
-DCMAKE_BUILD_TYPE={BUILD_TYPE} \
-DSANITIZE={SANITIZER} \
-DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 \
-DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr \
-DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \
{AUX_DEFS} \
-DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 \
-DCOMPILER_CACHE={CACHE_TYPE} \
-DENABLE_BUILD_PROFILING=1 {DIR}"""
def main():
args = parse_args()
@ -52,41 +33,23 @@ def main():
stages.pop(0)
stages.insert(0, stage)
build_type = args.build_type
assert (
build_type
), "build_type must be provided either as input argument or as a parameter of parametrized job in CI"
build_type = build_type.lower()
cmake_build_type = "Release"
sanitizer = ""
CACHE_TYPE = "sccache"
BUILD_TYPE = "RelWithDebInfo"
SANITIZER = ""
AUX_DEFS = " -DENABLE_TESTS=0 "
if "debug" in build_type:
if "debug" in args.BUILD_TYPE.lower():
print("Build type set: debug")
BUILD_TYPE = "Debug"
AUX_DEFS = " -DENABLE_TESTS=1 "
elif "release" in build_type:
print("Build type set: release")
AUX_DEFS = (
" -DENABLE_TESTS=0 -DSPLIT_DEBUG_SYMBOLS=ON -DBUILD_STANDALONE_KEEPER=1 "
)
elif "asan" in build_type:
cmake_build_type = "Debug"
if "asan" in args.BUILD_TYPE.lower():
print("Sanitizer set: address")
SANITIZER = "address"
else:
assert False
sanitizer = "address"
cmake_cmd = CMAKE_CMD.format(
BUILD_TYPE=BUILD_TYPE,
CACHE_TYPE=CACHE_TYPE,
SANITIZER=SANITIZER,
AUX_DEFS=AUX_DEFS,
DIR=Utils.cwd(),
)
# if Environment.is_local_run():
# build_cache_type = "disabled"
# else:
build_cache_type = "sccache"
current_directory = Utils.cwd()
build_dir = f"{Settings.TEMP_DIR}/build"
res = True
@ -106,7 +69,12 @@ def main():
results.append(
Result.create_from_command_execution(
name="Cmake configuration",
command=cmake_cmd,
command=f"cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA -DCMAKE_BUILD_TYPE={cmake_build_type} \
-DSANITIZE={sanitizer} -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_CLICKHOUSE_SELF_EXTRACTING=1 -DENABLE_TESTS=0 \
-DENABLE_UTILS=0 -DCMAKE_FIND_PACKAGE_NO_PACKAGE_REGISTRY=ON -DCMAKE_INSTALL_PREFIX=/usr \
-DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \
-DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 -DCOMPILER_CACHE={build_cache_type} -DENABLE_TESTS=1 \
-DENABLE_BUILD_PROFILING=1 {current_directory}",
workdir=build_dir,
with_log=True,
)
@ -127,7 +95,7 @@ def main():
Shell.check(f"ls -l {build_dir}/programs/")
res = results[-1].is_ok()
Result.create_from(results=results, stopwatch=stop_watch).complete_job()
Result.create_from(results=results, stopwatch=stop_watch).finish_job_accordingly()
if __name__ == "__main__":

2
ci/jobs/check_style.py
View File

@ -379,4 +379,4 @@ if __name__ == "__main__":
)
)
Result.create_from(results=results, stopwatch=stop_watch).complete_job()
Result.create_from(results=results, stopwatch=stop_watch).finish_job_accordingly()

121
ci/jobs/fast_test.py
View File

@ -1,13 +1,120 @@
import argparse
import threading
from pathlib import Path
from praktika.result import Result
from praktika.settings import Settings
from praktika.utils import MetaClasses, Shell, Utils
from ci.jobs.scripts.clickhouse_proc import ClickHouseProc
from ci.jobs.scripts.functional_tests_results import FTResultsProcessor
class ClickHouseProc:
def __init__(self):
self.ch_config_dir = f"{Settings.TEMP_DIR}/etc/clickhouse-server"
self.pid_file = f"{self.ch_config_dir}/clickhouse-server.pid"
self.config_file = f"{self.ch_config_dir}/config.xml"
self.user_files_path = f"{self.ch_config_dir}/user_files"
self.test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt"
self.command = f"clickhouse-server --config-file {self.config_file} --pid-file {self.pid_file} -- --path {self.ch_config_dir} --user_files_path {self.user_files_path} --top_level_domains_path {self.ch_config_dir}/top_level_domains --keeper_server.storage_path {self.ch_config_dir}/coordination"
self.proc = None
self.pid = 0
nproc = int(Utils.cpu_count() / 2)
self.fast_test_command = f"clickhouse-test --hung-check --fast-tests-only --no-random-settings --no-random-merge-tree-settings --no-long --testname --shard --zookeeper --check-zookeeper-session --order random --print-time --report-logs-stats --jobs {nproc} -- '' | ts '%Y-%m-%d %H:%M:%S' \
| tee -a \"{self.test_output_file}\""
# TODO: store info in case of failure
self.info = ""
self.info_file = ""
Utils.set_env("CLICKHOUSE_CONFIG_DIR", self.ch_config_dir)
Utils.set_env("CLICKHOUSE_CONFIG", self.config_file)
Utils.set_env("CLICKHOUSE_USER_FILES", self.user_files_path)
Utils.set_env("CLICKHOUSE_SCHEMA_FILES", f"{self.ch_config_dir}/format_schemas")
def start(self):
print("Starting ClickHouse server")
Shell.check(f"rm {self.pid_file}")
def run_clickhouse():
self.proc = Shell.run_async(
self.command, verbose=True, suppress_output=True
)
thread = threading.Thread(target=run_clickhouse)
thread.daemon = True # Allow program to exit even if thread is still running
thread.start()
# self.proc = Shell.run_async(self.command, verbose=True)
started = False
try:
for _ in range(5):
pid = Shell.get_output(f"cat {self.pid_file}").strip()
if not pid:
Utils.sleep(1)
continue
started = True
print(f"Got pid from fs [{pid}]")
_ = int(pid)
break
except Exception:
pass
if not started:
stdout = self.proc.stdout.read().strip() if self.proc.stdout else ""
stderr = self.proc.stderr.read().strip() if self.proc.stderr else ""
Utils.print_formatted_error("Failed to start ClickHouse", stdout, stderr)
return False
print(f"ClickHouse server started successfully, pid [{pid}]")
return True
def wait_ready(self):
res, out, err = 0, "", ""
attempts = 30
delay = 2
for attempt in range(attempts):
res, out, err = Shell.get_res_stdout_stderr(
'clickhouse-client --query "select 1"', verbose=True
)
if out.strip() == "1":
print("Server ready")
break
else:
print(f"Server not ready, wait")
Utils.sleep(delay)
else:
Utils.print_formatted_error(
f"Server not ready after [{attempts*delay}s]", out, err
)
return False
return True
def run_fast_test(self):
if Path(self.test_output_file).exists():
Path(self.test_output_file).unlink()
exit_code = Shell.run(self.fast_test_command)
return exit_code == 0
def terminate(self):
print("Terminate ClickHouse process")
timeout = 10
if self.proc:
Utils.terminate_process_group(self.proc.pid)
self.proc.terminate()
try:
self.proc.wait(timeout=10)
print(f"Process {self.proc.pid} terminated gracefully.")
except Exception:
print(
f"Process {self.proc.pid} did not terminate in {timeout} seconds, killing it..."
)
Utils.terminate_process_group(self.proc.pid, force=True)
self.proc.wait() # Wait for the process to be fully killed
print(f"Process {self.proc} was killed.")
def clone_submodules():
submodules_to_update = [
"contrib/sysroot",
@ -133,7 +240,7 @@ def main():
Shell.check(f"rm -rf {build_dir} && mkdir -p {build_dir}")
results.append(
Result.create_from_command_execution(
name="Checkout Submodules",
name="Checkout Submodules for Minimal Build",
command=clone_submodules,
)
)
@ -188,8 +295,8 @@ def main():
if res and JobStages.CONFIG in stages:
commands = [
f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
f"cp ./programs/server/config.xml ./programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --fast-test",
f"cp {current_directory}/programs/server/config.xml {current_directory}/programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
f"{current_directory}/tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client",
# f"cp -a {current_directory}/programs/server/config.d/log_to_console.xml {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/",
f"rm -f {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/secure_ports.xml",
update_path_ch_config,
@ -203,7 +310,7 @@ def main():
)
res = results[-1].is_ok()
CH = ClickHouseProc(fast_test=True)
CH = ClickHouseProc()
if res and JobStages.TEST in stages:
stop_watch_ = Utils.Stopwatch()
step_name = "Start ClickHouse Server"
@ -215,17 +322,15 @@ def main():
)
if res and JobStages.TEST in stages:
stop_watch_ = Utils.Stopwatch()
step_name = "Tests"
print(step_name)
res = res and CH.run_fast_test()
if res:
results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
results[-1].set_timing(stopwatch=stop_watch_)
CH.terminate()
Result.create_from(results=results, stopwatch=stop_watch).complete_job()
Result.create_from(results=results, stopwatch=stop_watch).finish_job_accordingly()
if __name__ == "__main__":

170
ci/jobs/functional_stateful_tests.py
View File

@ -1,170 +0,0 @@
import argparse
import os
import time
from pathlib import Path
from praktika.result import Result
from praktika.settings import Settings
from praktika.utils import MetaClasses, Shell, Utils
from ci.jobs.scripts.clickhouse_proc import ClickHouseProc
from ci.jobs.scripts.functional_tests_results import FTResultsProcessor
class JobStages(metaclass=MetaClasses.WithIter):
INSTALL_CLICKHOUSE = "install"
START = "start"
TEST = "test"
def parse_args():
parser = argparse.ArgumentParser(description="ClickHouse Build Job")
parser.add_argument(
"--ch-path", help="Path to clickhouse binary", default=f"{Settings.INPUT_DIR}"
)
parser.add_argument(
"--test-options",
help="Comma separated option(s): parallel|non-parallel|BATCH_NUM/BTATCH_TOT|..",
default="",
)
parser.add_argument("--param", help="Optional job start stage", default=None)
parser.add_argument("--test", help="Optional test name pattern", default="")
return parser.parse_args()
def run_test(
no_parallel: bool, no_sequiential: bool, batch_num: int, batch_total: int, test=""
):
test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt"
test_command = f"clickhouse-test --jobs 2 --testname --shard --zookeeper --check-zookeeper-session --no-stateless \
--hung-check --print-time \
--capture-client-stacktrace --queries ./tests/queries -- '{test}' \
| ts '%Y-%m-%d %H:%M:%S' | tee -a \"{test_output_file}\""
if Path(test_output_file).exists():
Path(test_output_file).unlink()
Shell.run(test_command, verbose=True)
def main():
args = parse_args()
test_options = args.test_options.split(",")
no_parallel = "non-parallel" in test_options
no_sequential = "parallel" in test_options
batch_num, total_batches = 0, 0
for to in test_options:
if "/" in to:
batch_num, total_batches = map(int, to.split("/"))
# os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output(
# f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value",
# verbose=True,
# strict=True
# )
ch_path = args.ch_path
assert Path(
ch_path + "/clickhouse"
).is_file(), f"clickhouse binary not found under [{ch_path}]"
stop_watch = Utils.Stopwatch()
stages = list(JobStages)
logs_to_attach = []
stage = args.param or JobStages.INSTALL_CLICKHOUSE
if stage:
assert stage in JobStages, f"--param must be one of [{list(JobStages)}]"
print(f"Job will start from stage [{stage}]")
while stage in stages:
stages.pop(0)
stages.insert(0, stage)
res = True
results = []
Utils.add_to_PATH(f"{ch_path}:tests")
if res and JobStages.INSTALL_CLICKHOUSE in stages:
commands = [
f"rm -rf /tmp/praktika/var/log/clickhouse-server/clickhouse-server.*",
f"chmod +x {ch_path}/clickhouse",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-server",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local",
f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage",
# clickhouse benchmark segfaults with --config-path, so provide client config by its default location
f"cp {Settings.TEMP_DIR}/etc/clickhouse-client/* /etc/clickhouse-client/",
# update_path_ch_config,
# f"sed -i 's|>/var/|>{Settings.TEMP_DIR}/var/|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.xml",
# f"sed -i 's|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/ssl_certs.xml",
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|<path>local_disk|<path>{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done",
f"clickhouse-server --version",
]
results.append(
Result.create_from_command_execution(
name="Install ClickHouse", command=commands, with_log=True
)
)
res = results[-1].is_ok()
CH = ClickHouseProc()
if res and JobStages.START in stages:
stop_watch_ = Utils.Stopwatch()
step_name = "Start ClickHouse Server"
print(step_name)
minio_log = "/tmp/praktika/output/minio.log"
res = res and CH.start_minio(test_type="stateful", log_file_path=minio_log)
logs_to_attach += [minio_log]
time.sleep(10)
Shell.check("ps -ef | grep minio", verbose=True)
res = res and Shell.check(
"aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True
)
res = res and CH.start()
res = res and CH.wait_ready()
if res:
print("ch started")
logs_to_attach += [
"/tmp/praktika/var/log/clickhouse-server/clickhouse-server.log",
"/tmp/praktika/var/log/clickhouse-server/clickhouse-server.err.log",
]
results.append(
Result.create_from(
name=step_name,
status=res,
stopwatch=stop_watch_,
)
)
res = results[-1].is_ok()
if res and JobStages.TEST in stages:
stop_watch_ = Utils.Stopwatch()
step_name = "Tests"
print(step_name)
# assert Shell.check("clickhouse-client -q \"insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')\"", verbose=True)
run_test(
no_parallel=no_parallel,
no_sequiential=no_sequential,
batch_num=batch_num,
batch_total=total_batches,
test=args.test,
)
results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
results[-1].set_timing(stopwatch=stop_watch_)
res = results[-1].is_ok()
Result.create_from(
results=results, stopwatch=stop_watch, files=logs_to_attach if not res else []
).complete_job()
if __name__ == "__main__":
main()

183
ci/jobs/functional_stateless_tests.py
View File

@ -1,183 +0,0 @@
import argparse
import os
import time
from pathlib import Path
from praktika.result import Result
from praktika.settings import Settings
from praktika.utils import MetaClasses, Shell, Utils
from ci.jobs.scripts.clickhouse_proc import ClickHouseProc
from ci.jobs.scripts.functional_tests_results import FTResultsProcessor
class JobStages(metaclass=MetaClasses.WithIter):
INSTALL_CLICKHOUSE = "install"
START = "start"
TEST = "test"
def parse_args():
parser = argparse.ArgumentParser(description="ClickHouse Build Job")
parser.add_argument(
"--ch-path", help="Path to clickhouse binary", default=f"{Settings.INPUT_DIR}"
)
parser.add_argument(
"--test-options",
help="Comma separated option(s): parallel|non-parallel|BATCH_NUM/BTATCH_TOT|..",
default="",
)
parser.add_argument("--param", help="Optional job start stage", default=None)
parser.add_argument("--test", help="Optional test name pattern", default="")
return parser.parse_args()
def run_stateless_test(
no_parallel: bool, no_sequiential: bool, batch_num: int, batch_total: int, test=""
):
assert not (no_parallel and no_sequiential)
test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt"
aux = ""
nproc = int(Utils.cpu_count() / 2)
if batch_num and batch_total:
aux = f"--run-by-hash-total {batch_total} --run-by-hash-num {batch_num-1}"
statless_test_command = f"clickhouse-test --testname --shard --zookeeper --check-zookeeper-session --hung-check --print-time \
--no-drop-if-fail --capture-client-stacktrace --queries /repo/tests/queries --test-runs 1 --hung-check \
{'--no-parallel' if no_parallel else ''} {'--no-sequential' if no_sequiential else ''} \
--print-time --jobs {nproc} --report-coverage --report-logs-stats {aux} \
--queries ./tests/queries -- '{test}' | ts '%Y-%m-%d %H:%M:%S' \
| tee -a \"{test_output_file}\""
if Path(test_output_file).exists():
Path(test_output_file).unlink()
Shell.run(statless_test_command, verbose=True)
def main():
args = parse_args()
test_options = args.test_options.split(",")
no_parallel = "non-parallel" in test_options
no_sequential = "parallel" in test_options
batch_num, total_batches = 0, 0
for to in test_options:
if "/" in to:
batch_num, total_batches = map(int, to.split("/"))
# os.environ["AZURE_CONNECTION_STRING"] = Shell.get_output(
# f"aws ssm get-parameter --region us-east-1 --name azure_connection_string --with-decryption --output text --query Parameter.Value",
# verbose=True,
# strict=True
# )
ch_path = args.ch_path
assert Path(
ch_path + "/clickhouse"
).is_file(), f"clickhouse binary not found under [{ch_path}]"
stop_watch = Utils.Stopwatch()
stages = list(JobStages)
logs_to_attach = []
stage = args.param or JobStages.INSTALL_CLICKHOUSE
if stage:
assert stage in JobStages, f"--param must be one of [{list(JobStages)}]"
print(f"Job will start from stage [{stage}]")
while stage in stages:
stages.pop(0)
stages.insert(0, stage)
res = True
results = []
Utils.add_to_PATH(f"{ch_path}:tests")
if res and JobStages.INSTALL_CLICKHOUSE in stages:
commands = [
f"rm -rf /tmp/praktika/var/log/clickhouse-server/clickhouse-server.*",
f"chmod +x {ch_path}/clickhouse",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-server",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local",
f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
# TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
f"./tests/config/install.sh {Settings.TEMP_DIR}/etc/clickhouse-server {Settings.TEMP_DIR}/etc/clickhouse-client --s3-storage --no-azure",
# clickhouse benchmark segfaults with --config-path, so provide client config by its default location
f"cp {Settings.TEMP_DIR}/etc/clickhouse-client/* /etc/clickhouse-client/",
# update_path_ch_config,
# f"sed -i 's|>/var/|>{Settings.TEMP_DIR}/var/|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.xml",
# f"sed -i 's|>/etc/|>{Settings.TEMP_DIR}/etc/|g' {Settings.TEMP_DIR}/etc/clickhouse-server/config.d/ssl_certs.xml",
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|<path>local_disk|<path>{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done",
f"clickhouse-server --version",
]
results.append(
Result.create_from_command_execution(
name="Install ClickHouse", command=commands, with_log=True
)
)
res = results[-1].is_ok()
CH = ClickHouseProc()
if res and JobStages.START in stages:
stop_watch_ = Utils.Stopwatch()
step_name = "Start ClickHouse Server"
print(step_name)
hdfs_log = "/tmp/praktika/output/hdfs_mini.log"
minio_log = "/tmp/praktika/output/minio.log"
res = res and CH.start_hdfs(log_file_path=hdfs_log)
res = res and CH.start_minio(test_type="stateful", log_file_path=minio_log)
logs_to_attach += [minio_log, hdfs_log]
time.sleep(10)
Shell.check("ps -ef | grep minio", verbose=True)
Shell.check("ps -ef | grep hdfs", verbose=True)
res = res and Shell.check(
"aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True
)
res = res and CH.start()
res = res and CH.wait_ready()
if res:
print("ch started")
logs_to_attach += [
"/tmp/praktika/var/log/clickhouse-server/clickhouse-server.log",
"/tmp/praktika/var/log/clickhouse-server/clickhouse-server.err.log",
]
results.append(
Result.create_from(
name=step_name,
status=res,
stopwatch=stop_watch_,
)
)
res = results[-1].is_ok()
if res and JobStages.TEST in stages:
stop_watch_ = Utils.Stopwatch()
step_name = "Tests"
print(step_name)
assert Shell.check(
"clickhouse-client -q \"insert into system.zookeeper (name, path, value) values ('auxiliary_zookeeper2', '/test/chroot/', '')\"",
verbose=True,
)
run_stateless_test(
no_parallel=no_parallel,
no_sequiential=no_sequential,
batch_num=batch_num,
batch_total=total_batches,
test=args.test,
)
results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
results[-1].set_timing(stopwatch=stop_watch_)
res = results[-1].is_ok()
Result.create_from(
results=results, stopwatch=stop_watch, files=logs_to_attach if not res else []
).complete_job()
if __name__ == "__main__":
main()

142
ci/jobs/scripts/clickhouse_proc.py
View File

@ -1,142 +0,0 @@
import subprocess
from pathlib import Path
from praktika.settings import Settings
from praktika.utils import Shell, Utils
class ClickHouseProc:
BACKUPS_XML = """
<clickhouse>
<backups>
<type>local</type>
<path>{CH_RUNTIME_DIR}/var/lib/clickhouse/disks/backups/</path>
</backups>
</clickhouse>
"""
def __init__(self, fast_test=False):
self.ch_config_dir = f"{Settings.TEMP_DIR}/etc/clickhouse-server"
self.pid_file = f"{self.ch_config_dir}/clickhouse-server.pid"
self.config_file = f"{self.ch_config_dir}/config.xml"
self.user_files_path = f"{self.ch_config_dir}/user_files"
self.test_output_file = f"{Settings.OUTPUT_DIR}/test_result.txt"
self.command = f"clickhouse-server --config-file {self.config_file} --pid-file {self.pid_file} -- --path {self.ch_config_dir} --user_files_path {self.user_files_path} --top_level_domains_path {self.ch_config_dir}/top_level_domains --keeper_server.storage_path {self.ch_config_dir}/coordination"
self.proc = None
self.pid = 0
nproc = int(Utils.cpu_count() / 2)
self.fast_test_command = f"clickhouse-test --hung-check --fast-tests-only --no-random-settings --no-random-merge-tree-settings --no-long --testname --shard --zookeeper --check-zookeeper-session --order random --print-time --report-logs-stats --jobs {nproc} -- '' | ts '%Y-%m-%d %H:%M:%S' \
| tee -a \"{self.test_output_file}\""
# TODO: store info in case of failure
self.info = ""
self.info_file = ""
Utils.set_env("CLICKHOUSE_CONFIG_DIR", self.ch_config_dir)
Utils.set_env("CLICKHOUSE_CONFIG", self.config_file)
Utils.set_env("CLICKHOUSE_USER_FILES", self.user_files_path)
# Utils.set_env("CLICKHOUSE_SCHEMA_FILES", f"{self.ch_config_dir}/format_schemas")
# if not fast_test:
# with open(f"{self.ch_config_dir}/config.d/backups.xml", "w") as file:
# file.write(self.BACKUPS_XML)
self.minio_proc = None
def start_hdfs(self, log_file_path):
command = ["./ci/jobs/scripts/functional_tests/setup_hdfs_minicluster.sh"]
with open(log_file_path, "w") as log_file:
process = subprocess.Popen(
command, stdout=log_file, stderr=subprocess.STDOUT
)
print(
f"Started setup_hdfs_minicluster.sh asynchronously with PID {process.pid}"
)
return True
def start_minio(self, test_type, log_file_path):
command = [
"./ci/jobs/scripts/functional_tests/setup_minio.sh",
test_type,
"./tests",
]
with open(log_file_path, "w") as log_file:
process = subprocess.Popen(
command, stdout=log_file, stderr=subprocess.STDOUT
)
print(f"Started setup_minio.sh asynchronously with PID {process.pid}")
return True
def start(self):
print("Starting ClickHouse server")
Shell.check(f"rm {self.pid_file}")
self.proc = subprocess.Popen(self.command, stderr=subprocess.STDOUT, shell=True)
started = False
try:
for _ in range(5):
pid = Shell.get_output(f"cat {self.pid_file}").strip()
if not pid:
Utils.sleep(1)
continue
started = True
print(f"Got pid from fs [{pid}]")
_ = int(pid)
break
except Exception:
pass
if not started:
stdout = self.proc.stdout.read().strip() if self.proc.stdout else ""
stderr = self.proc.stderr.read().strip() if self.proc.stderr else ""
Utils.print_formatted_error("Failed to start ClickHouse", stdout, stderr)
return False
print(f"ClickHouse server started successfully, pid [{pid}]")
return True
def wait_ready(self):
res, out, err = 0, "", ""
attempts = 30
delay = 2
for attempt in range(attempts):
res, out, err = Shell.get_res_stdout_stderr(
'clickhouse-client --query "select 1"', verbose=True
)
if out.strip() == "1":
print("Server ready")
break
else:
print(f"Server not ready, wait")
Utils.sleep(delay)
else:
Utils.print_formatted_error(
f"Server not ready after [{attempts*delay}s]", out, err
)
return False
return True
def run_fast_test(self):
if Path(self.test_output_file).exists():
Path(self.test_output_file).unlink()
exit_code = Shell.run(self.fast_test_command)
return exit_code == 0
def terminate(self):
print("Terminate ClickHouse process")
timeout = 10
if self.proc:
Utils.terminate_process_group(self.proc.pid)
self.proc.terminate()
try:
self.proc.wait(timeout=10)
print(f"Process {self.proc.pid} terminated gracefully.")
except Exception:
print(
f"Process {self.proc.pid} did not terminate in {timeout} seconds, killing it..."
)
Utils.terminate_process_group(self.proc.pid, force=True)
self.proc.wait() # Wait for the process to be fully killed
print(f"Process {self.proc} was killed.")
if self.minio_proc:
Utils.terminate_process_group(self.minio_proc.pid)

19
ci/jobs/scripts/functional_tests/setup_hdfs_minicluster.sh
View File

@ -1,19 +0,0 @@
#!/bin/bash
# shellcheck disable=SC2024
set -e -x -a -u
ls -lha
cd /hadoop-3.3.1
export JAVA_HOME=/usr
mkdir -p target/test/data
bin/mapred minicluster -format -nomr -nnport 12222 &
while ! nc -z localhost 12222; do
sleep 1
done
lsof -i :12222

162
ci/jobs/scripts/functional_tests/setup_minio.sh
View File

@ -1,162 +0,0 @@
#!/bin/bash
set -euxf -o pipefail
export MINIO_ROOT_USER=${MINIO_ROOT_USER:-clickhouse}
export MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD:-clickhouse}
TEST_DIR=${2:-/repo/tests/}
if [ -d "$TEMP_DIR" ]; then
TEST_DIR=$(readlink -f $TEST_DIR)
cd "$TEMP_DIR"
# add / for minio mc in docker
PATH="/:.:$PATH"
fi
usage() {
echo $"Usage: $0 <stateful|stateless> <test_path> (default path: /usr/share/clickhouse-test)"
exit 1
}
check_arg() {
local query_dir
if [ ! $# -eq 1 ]; then
if [ ! $# -eq 2 ]; then
echo "ERROR: need either one or two arguments, <stateful|stateless> <test_path> (default path: /usr/share/clickhouse-test)"
usage
fi
fi
case "$1" in
stateless)
query_dir="0_stateless"
;;
stateful)
query_dir="1_stateful"
;;
*)
echo "unknown test type ${test_type}"
usage
;;
esac
echo ${query_dir}
}
find_arch() {
local arch
case $(uname -m) in
x86_64)
arch="amd64"
;;
aarch64)
arch="arm64"
;;
*)
echo "unknown architecture $(uname -m)";
exit 1
;;
esac
echo ${arch}
}
find_os() {
local os
os=$(uname -s | tr '[:upper:]' '[:lower:]')
echo "${os}"
}
download_minio() {
local os
local arch
local minio_server_version=${MINIO_SERVER_VERSION:-2024-08-03T04-33-23Z}
local minio_client_version=${MINIO_CLIENT_VERSION:-2024-07-31T15-58-33Z}
os=$(find_os)
arch=$(find_arch)
wget "https://dl.min.io/server/minio/release/${os}-${arch}/archive/minio.RELEASE.${minio_server_version}" -O ./minio
wget "https://dl.min.io/client/mc/release/${os}-${arch}/archive/mc.RELEASE.${minio_client_version}" -O ./mc
chmod +x ./mc ./minio
}
start_minio() {
pwd
mkdir -p ./minio_data
minio --version
nohup minio server --address ":11111" ./minio_data &
wait_for_it
lsof -i :11111
sleep 5
}
setup_minio() {
local test_type=$1
echo "setup_minio(), test_type=$test_type"
mc alias set clickminio http://localhost:11111 clickhouse clickhouse
mc admin user add clickminio test testtest
mc admin policy attach clickminio readwrite --user=test ||:
mc mb --ignore-existing clickminio/test
if [ "$test_type" = "stateless" ]; then
echo "Create @test bucket in minio"
mc anonymous set public clickminio/test
fi
}
# uploads data to minio, by default after unpacking all tests
# will be in /usr/share/clickhouse-test/queries
upload_data() {
local query_dir=$1
local test_path=$2
local data_path=${test_path}/queries/${query_dir}/data_minio
echo "upload_data() data_path=$data_path"
# iterating over globs will cause redundant file variable to be
# a path to a file, not a filename
# shellcheck disable=SC2045
if [ -d "${data_path}" ]; then
mc cp --recursive "${data_path}"/ clickminio/test/
fi
}
setup_aws_credentials() {
local minio_root_user=${MINIO_ROOT_USER:-clickhouse}
local minio_root_password=${MINIO_ROOT_PASSWORD:-clickhouse}
mkdir -p ~/.aws
cat <<EOT >> ~/.aws/credentials
[default]
aws_access_key_id=${minio_root_user}
aws_secret_access_key=${minio_root_password}
EOT
}
wait_for_it() {
local counter=0
local max_counter=60
local url="http://localhost:11111"
local params=(
--silent
--verbose
)
while ! curl "${params[@]}" "${url}" 2>&1 | grep AccessDenied
do
if [[ ${counter} == "${max_counter}" ]]; then
echo "failed to setup minio"
exit 0
fi
echo "trying to connect to minio"
sleep 1
counter=$((counter + 1))
done
}
main() {
local query_dir
query_dir=$(check_arg "$@")
if ! (minio --version && mc --version); then
download_minio
fi
start_minio
setup_minio "$1"
upload_data "${query_dir}" "$TEST_DIR"
setup_aws_credentials
}
main "$@"

6
ci/jobs/scripts/functional_tests_results.py
View File

@ -1,6 +1,7 @@
import dataclasses
from typing import List
from praktika.environment import Environment
from praktika.result import Result
OK_SIGN = "[ OK "
@ -232,8 +233,6 @@ class FTResultsProcessor:
else:
pass
info = f"Total: {s.total - s.skipped}, Failed: {s.failed}"
# TODO: !!!
# def test_result_comparator(item):
# # sort by status then by check name
@ -251,11 +250,10 @@ class FTResultsProcessor:
# test_results.sort(key=test_result_comparator)
return Result.create_from(
name="Tests",
name=Environment.JOB_NAME,
results=test_results,
status=state,
files=[self.tests_output_file],
info=info,
with_info_from_results=False,
)

30
ci/praktika/__main__.py
View File

@ -37,30 +37,6 @@ def create_parser():
type=str,
default=None,
)
run_parser.add_argument(
"--test",
help="Custom parameter to pass into a job script, it's up to job script how to use it, for local test",
type=str,
default="",
)
run_parser.add_argument(
"--pr",
help="PR number. Optional parameter for local run. Set if you want an required artifact to be uploaded from CI run in that PR",
type=int,
default=None,
)
run_parser.add_argument(
"--sha",
help="Commit sha. Optional parameter for local run. Set if you want an required artifact to be uploaded from CI run on that sha, head sha will be used if not set",
type=str,
default=None,
)
run_parser.add_argument(
"--branch",
help="Commit sha. Optional parameter for local run. Set if you want an required artifact to be uploaded from CI run on that branch, main branch name will be used if not set",
type=str,
default=None,
)
run_parser.add_argument(
"--ci",
help="When not set - dummy env will be generated, for local test",
@ -109,13 +85,9 @@ if __name__ == "__main__":
workflow=workflow,
job=job,
docker=args.docker,
local_run=not args.ci,
dummy_env=not args.ci,
no_docker=args.no_docker,
param=args.param,
test=args.test,
pr=args.pr,
branch=args.branch,
sha=args.sha,
)
else:
parser.print_help()

25
ci/praktika/_environment.py
View File

@ -6,7 +6,7 @@ from types import SimpleNamespace
from typing import Any, Dict, List, Type
from praktika import Workflow
from praktika.settings import Settings
from praktika._settings import _Settings
from praktika.utils import MetaClasses, T
@ -30,12 +30,13 @@ class _Environment(MetaClasses.Serializable):
INSTANCE_ID: str
INSTANCE_LIFE_CYCLE: str
LOCAL_RUN: bool = False
PARAMETER: Any = None
REPORT_INFO: List[str] = dataclasses.field(default_factory=list)
name = "environment"
@classmethod
def file_name_static(cls, _name=""):
return f"{Settings.TEMP_DIR}/{cls.name}.json"
return f"{_Settings.TEMP_DIR}/{cls.name}.json"
@classmethod
def from_dict(cls: Type[T], obj: Dict[str, Any]) -> T:
@ -66,12 +67,12 @@ class _Environment(MetaClasses.Serializable):
@staticmethod
def get_needs_statuses():
if Path(Settings.WORKFLOW_STATUS_FILE).is_file():
with open(Settings.WORKFLOW_STATUS_FILE, "r", encoding="utf8") as f:
if Path(_Settings.WORKFLOW_STATUS_FILE).is_file():
with open(_Settings.WORKFLOW_STATUS_FILE, "r", encoding="utf8") as f:
return json.load(f)
else:
print(
f"ERROR: Status file [{Settings.WORKFLOW_STATUS_FILE}] does not exist"
f"ERROR: Status file [{_Settings.WORKFLOW_STATUS_FILE}] does not exist"
)
raise RuntimeError()
@ -158,8 +159,7 @@ class _Environment(MetaClasses.Serializable):
@classmethod
def get_s3_prefix_static(cls, pr_number, branch, sha, latest=False):
prefix = ""
assert sha or latest
if pr_number and pr_number > 0:
if pr_number > 0:
prefix += f"{pr_number}"
else:
prefix += f"{branch}"
@ -171,15 +171,18 @@ class _Environment(MetaClasses.Serializable):
# TODO: find a better place for the function. This file should not import praktika.settings
# as it's requires reading users config, that's why imports nested inside the function
def get_report_url(self, settings, latest=False):
def get_report_url(self):
import urllib
path = settings.HTML_S3_PATH
for bucket, endpoint in settings.S3_BUCKET_TO_HTTP_ENDPOINT.items():
from praktika.settings import Settings
from praktika.utils import Utils
path = Settings.HTML_S3_PATH
for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items():
if bucket in path:
path = path.replace(bucket, endpoint)
break
REPORT_URL = f"https://{path}/{Path(settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={'latest' if latest else self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}&name_1={urllib.parse.quote(self.JOB_NAME, safe='')}"
REPORT_URL = f"https://{path}/{Path(Settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}&name_1={urllib.parse.quote(self.JOB_NAME, safe='')}"
return REPORT_URL
def is_local_run(self):

124
ci/praktika/_settings.py Normal file
View File

@ -0,0 +1,124 @@
import dataclasses
from pathlib import Path
from typing import Dict, Iterable, List, Optional
@dataclasses.dataclass
class _Settings:
######################################
# Pipeline generation settings #
######################################
CI_PATH = "./ci"
WORKFLOW_PATH_PREFIX: str = "./.github/workflows"
WORKFLOWS_DIRECTORY: str = f"{CI_PATH}/workflows"
SETTINGS_DIRECTORY: str = f"{CI_PATH}/settings"
CI_CONFIG_JOB_NAME = "Config Workflow"
DOCKER_BUILD_JOB_NAME = "Docker Builds"
FINISH_WORKFLOW_JOB_NAME = "Finish Workflow"
READY_FOR_MERGE_STATUS_NAME = "Ready for Merge"
CI_CONFIG_RUNS_ON: Optional[List[str]] = None
DOCKER_BUILD_RUNS_ON: Optional[List[str]] = None
VALIDATE_FILE_PATHS: bool = True
######################################
# Runtime Settings #
######################################
MAX_RETRIES_S3 = 3
MAX_RETRIES_GH = 3
######################################
# S3 (artifact storage) settings #
######################################
S3_ARTIFACT_PATH: str = ""
######################################
# CI workspace settings #
######################################
TEMP_DIR: str = "/tmp/praktika"
OUTPUT_DIR: str = f"{TEMP_DIR}/output"
INPUT_DIR: str = f"{TEMP_DIR}/input"
PYTHON_INTERPRETER: str = "python3"
PYTHON_PACKET_MANAGER: str = "pip3"
PYTHON_VERSION: str = "3.9"
INSTALL_PYTHON_FOR_NATIVE_JOBS: bool = False
INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS: str = "./ci/requirements.txt"
ENVIRONMENT_VAR_FILE: str = f"{TEMP_DIR}/environment.json"
RUN_LOG: str = f"{TEMP_DIR}/praktika_run.log"
SECRET_GH_APP_ID: str = "GH_APP_ID"
SECRET_GH_APP_PEM_KEY: str = "GH_APP_PEM_KEY"
ENV_SETUP_SCRIPT: str = "/tmp/praktika_setup_env.sh"
WORKFLOW_STATUS_FILE: str = f"{TEMP_DIR}/workflow_status.json"
######################################
# CI Cache settings #
######################################
CACHE_VERSION: int = 1
CACHE_DIGEST_LEN: int = 20
CACHE_S3_PATH: str = ""
CACHE_LOCAL_PATH: str = f"{TEMP_DIR}/ci_cache"
######################################
# Report settings #
######################################
HTML_S3_PATH: str = ""
HTML_PAGE_FILE: str = "./praktika/json.html"
TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"])
S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None
DOCKERHUB_USERNAME: str = ""
DOCKERHUB_SECRET: str = ""
DOCKER_WD: str = "/wd"
######################################
# CI DB Settings #
######################################
SECRET_CI_DB_URL: str = "CI_DB_URL"
SECRET_CI_DB_PASSWORD: str = "CI_DB_PASSWORD"
CI_DB_DB_NAME = ""
CI_DB_TABLE_NAME = ""
CI_DB_INSERT_TIMEOUT_SEC = 5
_USER_DEFINED_SETTINGS = [
"S3_ARTIFACT_PATH",
"CACHE_S3_PATH",
"HTML_S3_PATH",
"S3_BUCKET_TO_HTTP_ENDPOINT",
"TEXT_CONTENT_EXTENSIONS",
"TEMP_DIR",
"OUTPUT_DIR",
"INPUT_DIR",
"CI_CONFIG_RUNS_ON",
"DOCKER_BUILD_RUNS_ON",
"CI_CONFIG_JOB_NAME",
"PYTHON_INTERPRETER",
"PYTHON_VERSION",
"PYTHON_PACKET_MANAGER",
"INSTALL_PYTHON_FOR_NATIVE_JOBS",
"INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS",
"MAX_RETRIES_S3",
"MAX_RETRIES_GH",
"VALIDATE_FILE_PATHS",
"DOCKERHUB_USERNAME",
"DOCKERHUB_SECRET",
"READY_FOR_MERGE_STATUS_NAME",
"SECRET_CI_DB_URL",
"SECRET_CI_DB_PASSWORD",
"CI_DB_DB_NAME",
"CI_DB_TABLE_NAME",
"CI_DB_INSERT_TIMEOUT_SEC",
"SECRET_GH_APP_PEM_KEY",
"SECRET_GH_APP_ID",
]
class GHRunners:
ubuntu = "ubuntu-latest"
if __name__ == "__main__":
for setting in _USER_DEFINED_SETTINGS:
print(_Settings().__getattribute__(setting))
# print(dataclasses.asdict(_Settings()))

2
ci/praktika/cidb.py
View File

@ -52,7 +52,7 @@ class CIDB:
check_status=result.status,
check_duration_ms=int(result.duration * 1000),
check_start_time=Utils.timestamp_to_str(result.start_time),
report_url=env.get_report_url(settings=Settings),
report_url=env.get_report_url(),
pull_request_url=env.CHANGE_URL,
base_ref=env.BASE_BRANCH,
base_repo=env.REPOSITORY,

38
ci/praktika/digest.py
View File

@ -23,7 +23,7 @@ class Digest:
hash_string = hash_obj.hexdigest()
return hash_string
def calc_job_digest(self, job_config: Job.Config, docker_digests):
def calc_job_digest(self, job_config: Job.Config):
config = job_config.digest_config
if not config:
return "f" * Settings.CACHE_DIGEST_LEN
@ -31,32 +31,32 @@ class Digest:
cache_key = self._hash_digest_config(config)
if cache_key in self.digest_cache:
print(
f"calc digest for job [{job_config.name}]: hash_key [{cache_key}] - from cache"
)
digest = self.digest_cache[cache_key]
else:
return self.digest_cache[cache_key]
included_files = Utils.traverse_paths(
job_config.digest_config.include_paths,
job_config.digest_config.exclude_paths,
sorted=True,
)
print(
f"calc digest for job [{job_config.name}]: hash_key [{cache_key}], include [{len(included_files)}] files"
)
# Sort files to ensure consistent hash calculation
included_files.sort()
# Calculate MD5 hash
res = ""
if not included_files:
res = "f" * Settings.CACHE_DIGEST_LEN
print(f"NOTE: empty digest config [{config}] - return dummy digest")
else:
hash_md5 = hashlib.md5()
for i, file_path in enumerate(included_files):
hash_md5 = self._calc_file_digest(file_path, hash_md5)
digest = hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]
self.digest_cache[cache_key] = digest
if job_config.run_in_docker:
# respect docker digest in the job digest
docker_digest = docker_digests[job_config.run_in_docker.split("+")[0]]
digest = "-".join([docker_digest, digest])
return digest
for file_path in included_files:
res = self._calc_file_digest(file_path, hash_md5)
assert res
self.digest_cache[cache_key] = res
return res
def calc_docker_digest(
self,
@ -103,10 +103,10 @@ class Digest:
print(
f"WARNING: No valid file resolved by link {file_path} -> {resolved_path} - skipping digest calculation"
)
return hash_md5
return hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]
with open(resolved_path, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5
return hash_md5.hexdigest()[: Settings.CACHE_DIGEST_LEN]

3
ci/praktika/environment.py Normal file
View File

@ -0,0 +1,3 @@
from praktika._environment import _Environment
Environment = _Environment.get()

2
ci/praktika/gh.py
View File

@ -18,7 +18,7 @@ class GH:
ret_code, out, err = Shell.get_res_stdout_stderr(command, verbose=True)
res = ret_code == 0
if not res and "Validation Failed" in err:
print(f"ERROR: GH command validation error.")
print("ERROR: GH command validation error")
break
if not res and "Bad credentials" in err:
print("ERROR: GH credentials/auth failure")

17
ci/praktika/hook_cache.py
View File

@ -1,5 +1,6 @@
from praktika._environment import _Environment
from praktika.cache import Cache
from praktika.mangle import _get_workflows
from praktika.runtime import RunConfig
from praktika.settings import Settings
from praktika.utils import Utils
@ -7,10 +8,11 @@ from praktika.utils import Utils
class CacheRunnerHooks:
@classmethod
def configure(cls, workflow):
workflow_config = RunConfig.from_fs(workflow.name)
docker_digests = workflow_config.digest_dockers
def configure(cls, _workflow):
workflow_config = RunConfig.from_fs(_workflow.name)
cache = Cache()
assert _Environment.get().WORKFLOW_NAME
workflow = _get_workflows(name=_Environment.get().WORKFLOW_NAME)[0]
print(f"Workflow Configure, workflow [{workflow.name}]")
assert (
workflow.enable_cache
@ -18,13 +20,11 @@ class CacheRunnerHooks:
artifact_digest_map = {}
job_digest_map = {}
for job in workflow.jobs:
digest = cache.digest.calc_job_digest(
job_config=job, docker_digests=docker_digests
)
if not job.digest_config:
print(
f"NOTE: job [{job.name}] has no Config.digest_config - skip cache check, always run"
)
digest = cache.digest.calc_job_digest(job_config=job)
job_digest_map[job.name] = digest
if job.provides:
# assign the job digest also to the artifacts it provides
@ -50,6 +50,7 @@ class CacheRunnerHooks:
), f"BUG, Workflow with enabled cache must have job digests after configuration, wf [{workflow.name}]"
print("Check remote cache")
job_to_cache_record = {}
for job_name, job_digest in workflow_config.digest_jobs.items():
record = cache.fetch_success(job_name=job_name, job_digest=job_digest)
if record:
@ -59,7 +60,7 @@ class CacheRunnerHooks:
)
workflow_config.cache_success.append(job_name)
workflow_config.cache_success_base64.append(Utils.to_base64(job_name))
workflow_config.cache_jobs[job_name] = record
job_to_cache_record[job_name] = record
print("Check artifacts to reuse")
for job in workflow.jobs:
@ -67,7 +68,7 @@ class CacheRunnerHooks:
if job.provides:
for artifact_name in job.provides:
workflow_config.cache_artifacts[artifact_name] = (
workflow_config.cache_jobs[job.name]
job_to_cache_record[job.name]
)
print(f"Write config to GH's job output")

193
ci/praktika/hook_html.py
View File

@ -1,125 +1,63 @@
import dataclasses
import json
import urllib.parse
from pathlib import Path
from typing import List
from praktika._environment import _Environment
from praktika.gh import GH
from praktika.parser import WorkflowConfigParser
from praktika.result import Result, ResultInfo, _ResultS3
from praktika.result import Result, ResultInfo
from praktika.runtime import RunConfig
from praktika.s3 import S3
from praktika.settings import Settings
from praktika.utils import Utils
from praktika.utils import Shell, Utils
@dataclasses.dataclass
class GitCommit:
# date: str
# message: str
date: str
message: str
sha: str
@staticmethod
def from_json(file) -> List["GitCommit"]:
def from_json(json_data: str) -> List["GitCommit"]:
commits = []
json_data = None
try:
with open(file, "r", encoding="utf-8") as f:
json_data = json.load(f)
data = json.loads(json_data)
commits = [
GitCommit(
# message=commit["messageHeadline"],
sha=commit["sha"],
# date=commit["committedDate"],
message=commit["messageHeadline"],
sha=commit["oid"],
date=commit["committedDate"],
)
for commit in json_data
for commit in data.get("commits", [])
]
except Exception as e:
print(
f"ERROR: Failed to deserialize commit's data [{json_data}], ex: [{e}]"
f"ERROR: Failed to deserialize commit's data: [{json_data}], ex: [{e}]"
)
return commits
@classmethod
def update_s3_data(cls):
env = _Environment.get()
sha = env.SHA
if not sha:
print("WARNING: Failed to retrieve commit sha")
return
commits = cls.pull_from_s3()
for commit in commits:
if sha == commit.sha:
print(
f"INFO: Sha already present in commits data [{sha}] - skip data update"
)
return
commits.append(GitCommit(sha=sha))
cls.push_to_s3(commits)
return
@classmethod
def dump(cls, commits):
commits_ = []
for commit in commits:
commits_.append(dataclasses.asdict(commit))
with open(cls.file_name(), "w", encoding="utf8") as f:
json.dump(commits_, f)
@classmethod
def pull_from_s3(cls):
local_path = Path(cls.file_name())
file_name = local_path.name
env = _Environment.get()
s3_path = f"{Settings.HTML_S3_PATH}/{cls.get_s3_prefix(pr_number=env.PR_NUMBER, branch=env.BRANCH)}/{file_name}"
if not S3.copy_file_from_s3(s3_path=s3_path, local_path=local_path):
print(f"WARNING: failed to cp file [{s3_path}] from s3")
return []
return cls.from_json(local_path)
@classmethod
def push_to_s3(cls, commits):
print(f"INFO: push commits data to s3, commits num [{len(commits)}]")
cls.dump(commits)
local_path = Path(cls.file_name())
file_name = local_path.name
env = _Environment.get()
s3_path = f"{Settings.HTML_S3_PATH}/{cls.get_s3_prefix(pr_number=env.PR_NUMBER, branch=env.BRANCH)}/{file_name}"
if not S3.copy_file_to_s3(s3_path=s3_path, local_path=local_path, text=True):
print(f"WARNING: failed to cp file [{local_path}] to s3")
@classmethod
def get_s3_prefix(cls, pr_number, branch):
prefix = ""
assert pr_number or branch
if pr_number and pr_number > 0:
prefix += f"{pr_number}"
else:
prefix += f"{branch}"
return prefix
@classmethod
def file_name(cls):
return f"{Settings.TEMP_DIR}/commits.json"
# def _get_pr_commits(pr_number):
# res = []
# if not pr_number:
# return res
# output = Shell.get_output(f"gh pr view {pr_number} --json commits")
# if output:
# res = GitCommit.from_json(output)
# return res
class HtmlRunnerHooks:
@classmethod
def configure(cls, _workflow):
def _get_pr_commits(pr_number):
res = []
if not pr_number:
return res
output = Shell.get_output(f"gh pr view {pr_number} --json commits")
if output:
res = GitCommit.from_json(output)
return res
# generate pending Results for all jobs in the workflow
if _workflow.enable_cache:
skip_jobs = RunConfig.from_fs(_workflow.name).cache_success
job_cache_records = RunConfig.from_fs(_workflow.name).cache_jobs
else:
skip_jobs = []
@ -129,22 +67,36 @@ class HtmlRunnerHooks:
if job.name not in skip_jobs:
result = Result.generate_pending(job.name)
else:
result = Result.generate_skipped(job.name, job_cache_records[job.name])
result = Result.generate_skipped(job.name)
results.append(result)
summary_result = Result.generate_pending(_workflow.name, results=results)
summary_result.links.append(env.CHANGE_URL)
summary_result.links.append(env.RUN_URL)
summary_result.aux_links.append(env.CHANGE_URL)
summary_result.aux_links.append(env.RUN_URL)
summary_result.start_time = Utils.timestamp()
page_url = "/".join(
["https:/", Settings.HTML_S3_PATH, str(Path(Settings.HTML_PAGE_FILE).name)]
)
for bucket, endpoint in Settings.S3_BUCKET_TO_HTTP_ENDPOINT.items():
page_url = page_url.replace(bucket, endpoint)
# TODO: add support for non-PRs (use branch?)
page_url += f"?PR={env.PR_NUMBER}&sha=latest&name_0={urllib.parse.quote(env.WORKFLOW_NAME, safe='')}"
summary_result.html_link = page_url
# clean the previous latest results in PR if any
if env.PR_NUMBER:
S3.clean_latest_result()
S3.copy_result_to_s3(
summary_result,
unlock=False,
)
assert _ResultS3.copy_result_to_s3_with_version(summary_result, version=0)
page_url = env.get_report_url(settings=Settings)
print(f"CI Status page url [{page_url}]")
res1 = GH.post_commit_status(
name=_workflow.name,
status=Result.Status.PENDING,
description="",
url=env.get_report_url(settings=Settings, latest=True),
url=page_url,
)
res2 = GH.post_pr_comment(
comment_body=f"Workflow [[{_workflow.name}]({page_url})], commit [{_Environment.get().SHA[:8]}]",
@ -154,15 +106,23 @@ class HtmlRunnerHooks:
Utils.raise_with_error(
"Failed to set both GH commit status and PR comment with Workflow Status, cannot proceed"
)
if env.PR_NUMBER:
# TODO: enable for branch, add commit number limiting
GitCommit.update_s3_data()
commits = _get_pr_commits(env.PR_NUMBER)
# TODO: upload commits data to s3 to visualise it on a report page
print(commits)
@classmethod
def pre_run(cls, _workflow, _job):
result = Result.from_fs(_job.name)
_ResultS3.update_workflow_results(
workflow_name=_workflow.name, new_sub_results=result
S3.copy_result_from_s3(
Result.file_name_static(_workflow.name),
)
workflow_result = Result.from_fs(_workflow.name)
workflow_result.update_sub_result(result)
S3.copy_result_to_s3(
workflow_result,
unlock=True,
)
@classmethod
@ -172,13 +132,14 @@ class HtmlRunnerHooks:
@classmethod
def post_run(cls, _workflow, _job, info_errors):
result = Result.from_fs(_job.name)
_ResultS3.upload_result_files_to_s3(result)
_ResultS3.copy_result_to_s3(result)
env = _Environment.get()
S3.copy_result_from_s3(
Result.file_name_static(_workflow.name),
lock=True,
)
workflow_result = Result.from_fs(_workflow.name)
print(f"Workflow info [{workflow_result.info}], info_errors [{info_errors}]")
new_sub_results = [result]
new_result_info = ""
env_info = env.REPORT_INFO
if env_info:
print(
@ -190,8 +151,14 @@ class HtmlRunnerHooks:
info_str = f"{_job.name}:\n"
info_str += "\n".join(info_errors)
print("Update workflow results with new info")
new_result_info = info_str
workflow_result.set_info(info_str)
old_status = workflow_result.status
S3.upload_result_files_to_s3(result)
workflow_result.update_sub_result(result)
skipped_job_results = []
if not result.is_ok():
print(
"Current job failed - find dependee jobs in the workflow and set their statuses to skipped"
@ -204,7 +171,7 @@ class HtmlRunnerHooks:
print(
f"NOTE: Set job [{dependee_job.name}] status to [{Result.Status.SKIPPED}] due to current failure"
)
new_sub_results.append(
skipped_job_results.append(
Result(
name=dependee_job.name,
status=Result.Status.SKIPPED,
@ -212,18 +179,20 @@ class HtmlRunnerHooks:
+ f" [{_job.name}]",
)
)
for skipped_job_result in skipped_job_results:
workflow_result.update_sub_result(skipped_job_result)
updated_status = _ResultS3.update_workflow_results(
new_info=new_result_info,
new_sub_results=new_sub_results,
workflow_name=_workflow.name,
S3.copy_result_to_s3(
workflow_result,
unlock=True,
)
if workflow_result.status != old_status:
print(
f"Update GH commit status [{result.name}]: [{old_status} -> {workflow_result.status}], link [{workflow_result.html_link}]"
)
if updated_status:
print(f"Update GH commit status [{result.name}]: [{updated_status}]")
GH.post_commit_status(
name=_workflow.name,
status=GH.convert_to_gh_status(updated_status),
name=workflow_result.name,
status=GH.convert_to_gh_status(workflow_result.status),
description="",
url=env.get_report_url(settings=Settings, latest=True),
url=workflow_result.html_link,
)

41
ci/praktika/job.py
View File

@ -52,58 +52,30 @@ class Job:
self,
parameter: Optional[List[Any]] = None,
runs_on: Optional[List[List[str]]] = None,
provides: Optional[List[List[str]]] = None,
requires: Optional[List[List[str]]] = None,
timeout: Optional[List[int]] = None,
):
assert (
parameter or runs_on
), "Either :parameter or :runs_on must be non empty list for parametrisation"
if runs_on:
assert isinstance(runs_on, list) and isinstance(runs_on[0], list)
if not parameter:
parameter = [None] * len(runs_on)
if not runs_on:
runs_on = [None] * len(parameter)
if not timeout:
timeout = [None] * len(parameter)
if not provides:
provides = [None] * len(parameter)
if not requires:
requires = [None] * len(parameter)
assert (
len(parameter)
== len(runs_on)
== len(timeout)
== len(provides)
== len(requires)
), f"Parametrization lists must be of the same size [{len(parameter)}, {len(runs_on)}, {len(timeout)}, {len(provides)}, {len(requires)}]"
len(parameter) == len(runs_on) == len(timeout)
), "Parametrization lists must be of the same size"
res = []
for parameter_, runs_on_, timeout_, provides_, requires_ in zip(
parameter, runs_on, timeout, provides, requires
):
for parameter_, runs_on_, timeout_ in zip(parameter, runs_on, timeout):
obj = copy.deepcopy(self)
assert (
not obj.provides
), "Job.Config.provides must be empty for parametrized jobs"
if parameter_:
obj.parameter = parameter_
obj.command = obj.command.format(PARAMETER=parameter_)
if runs_on_:
obj.runs_on = runs_on_
if timeout_:
obj.timeout = timeout_
if provides_:
assert (
not obj.provides
), "Job.Config.provides must be empty for parametrized jobs"
obj.provides = provides_
if requires_:
assert (
not obj.requires
), "Job.Config.requires and parametrize(requires=...) are both set"
obj.requires = requires_
obj.name = obj.get_job_name_with_parameter()
res.append(obj)
return res
@ -112,16 +84,13 @@ class Job:
name, parameter, runs_on = self.name, self.parameter, self.runs_on
res = name
name_params = []
if parameter:
if isinstance(parameter, list) or isinstance(parameter, dict):
name_params.append(json.dumps(parameter))
else:
elif parameter is not None:
name_params.append(parameter)
elif runs_on:
if runs_on:
assert isinstance(runs_on, list)
name_params.append(json.dumps(runs_on))
else:
assert False
if name_params:
name_params = [str(param) for param in name_params]
res += f" ({', '.join(name_params)})"

178
ci/praktika/json.html
View File

@ -89,27 +89,15 @@
letter-spacing: -0.5px;
}
.dropdown-value {
width: 100px;
font-weight: normal;
font-family: inherit;
background-color: transparent;
color: inherit;
/*border: none;*/
/*outline: none;*/
/*cursor: pointer;*/
}
#result-container {
background-color: var(--tile-background);
margin-left: calc(var(--status-width) + 20px);
padding: 0;
padding: 20px;
box-sizing: border-box;
text-align: center;
font-size: 18px;
font-weight: normal;
flex-grow: 1;
margin-bottom: 40px;
}
#footer {
@ -201,7 +189,10 @@
}
th.name-column, td.name-column {
min-width: 350px;
max-width: 400px; /* Set the maximum width for the column */
white-space: nowrap; /* Prevent text from wrapping */
overflow: hidden; /* Hide the overflowed text */
text-overflow: ellipsis; /* Show ellipsis (...) for overflowed text */
}
th.status-column, td.status-column {
@ -291,12 +282,6 @@
}
}
function updateUrlParameter(paramName, paramValue) {
const url = new URL(window.location.href);
url.searchParams.set(paramName, paramValue);
window.location.href = url.toString();
}
// Attach the toggle function to the click event of the icon
document.getElementById('theme-toggle').addEventListener('click', toggleTheme);
@ -306,14 +291,14 @@
const monthNames = ["Jan", "Feb", "Mar", "Apr", "May", "Jun",
"Jul", "Aug", "Sep", "Oct", "Nov", "Dec"];
const month = monthNames[date.getMonth()];
//const year = date.getFullYear();
const year = date.getFullYear();
const hours = String(date.getHours()).padStart(2, '0');
const minutes = String(date.getMinutes()).padStart(2, '0');
const seconds = String(date.getSeconds()).padStart(2, '0');
//const milliseconds = String(date.getMilliseconds()).padStart(2, '0');
return showDate
? `${day}'${month} ${hours}:${minutes}:${seconds}`
? `${day}-${month}-${year} ${hours}:${minutes}:${seconds}`
: `${hours}:${minutes}:${seconds}`;
}
@ -343,7 +328,7 @@
const milliseconds = Math.floor((duration % 1) * 1000);
const formattedSeconds = String(seconds);
const formattedMilliseconds = String(milliseconds).padStart(2, '0').slice(-2);
const formattedMilliseconds = String(milliseconds).padStart(3, '0');
return `${formattedSeconds}.${formattedMilliseconds}`;
}
@ -361,7 +346,8 @@
return 'status-other';
}
function addKeyValueToStatus(key, value, options = null) {
function addKeyValueToStatus(key, value) {
const statusContainer = document.getElementById('status-container');
let keyValuePair = document.createElement('div');
@ -371,40 +357,12 @@
keyElement.className = 'json-key';
keyElement.textContent = key + ':';
let valueElement;
if (options) {
// Create dropdown if options are provided
valueElement = document.createElement('select');
valueElement.className = 'dropdown-value';
options.forEach(optionValue => {
const option = document.createElement('option');
option.value = optionValue;
option.textContent = optionValue.slice(0, 10);
// Set the initially selected option
if (optionValue === value) {
option.selected = true;
}
valueElement.appendChild(option);
});
// Update the URL parameter when the selected value changes
valueElement.addEventListener('change', (event) => {
const selectedValue = event.target.value;
updateUrlParameter(key, selectedValue);
});
} else {
// Create a simple text display if no options are provided
valueElement = document.createElement('div');
const valueElement = document.createElement('div');
valueElement.className = 'json-value';
valueElement.textContent = value || 'N/A'; // Display 'N/A' if value is null
}
valueElement.textContent = value;
keyValuePair.appendChild(keyElement);
keyValuePair.appendChild(valueElement);
keyValuePair.appendChild(keyElement)
keyValuePair.appendChild(valueElement)
statusContainer.appendChild(keyValuePair);
}
@ -528,12 +486,12 @@
const columns = ['name', 'status', 'start_time', 'duration', 'info'];
const columnSymbols = {
name: '🗂️',
status: '🧾',
name: '📂',
status: '✔️',
start_time: '🕒',
duration: '⏳',
info: '📝',
files: '📎'
info: 'ℹ️',
files: '📄'
};
function createResultsTable(results, nest_level) {
@ -542,14 +500,16 @@
const thead = document.createElement('thead');
const tbody = document.createElement('tbody');
// Get the current URL parameters
const currentUrl = new URL(window.location.href);
// Create table headers based on the fixed columns
const headerRow = document.createElement('tr');
columns.forEach(column => {
const th = document.createElement('th');
th.textContent = columnSymbols[column] || column;
th.textContent = th.textContent = columnSymbols[column] || column;
th.style.cursor = 'pointer'; // Make headers clickable
th.setAttribute('data-sort-direction', 'asc'); // Default sort direction
th.addEventListener('click', () => sortTable(results, column, columnSymbols[column] || column, tbody, nest_level, columns)); // Add click event to sort the table
th.addEventListener('click', () => sortTable(results, column, tbody, nest_level)); // Add click event to sort the table
headerRow.appendChild(th);
});
thead.appendChild(headerRow);
@ -601,7 +561,8 @@
td.classList.add('time-column');
td.textContent = value ? formatDuration(value) : '';
} else if (column === 'info') {
td.textContent = value.includes('\n') ? '↵' : (value || '');
// For info and other columns, just display the value
td.textContent = value || '';
td.classList.add('info-column');
}
@ -612,33 +573,39 @@
});
}
function sortTable(results, column, key, tbody, nest_level, columns) {
function sortTable(results, key, tbody, nest_level) {
// Find the table header element for the given key
const tableHeaders = document.querySelectorAll('th');
let th = Array.from(tableHeaders).find(header => header.textContent === key);
let th = null;
const tableHeaders = document.querySelectorAll('th'); // Select all table headers
tableHeaders.forEach(header => {
if (header.textContent.trim().toLowerCase() === key.toLowerCase()) {
th = header;
}
});
if (!th) {
console.error(`No table header found for key: ${key}`);
return;
}
const ascending = th.getAttribute('data-sort-direction') === 'asc';
th.setAttribute('data-sort-direction', ascending ? 'desc' : 'asc');
// Determine the current sort direction
let ascending = th.getAttribute('data-sort-direction') === 'asc' ? false : true;
// Toggle the sort direction for the next click
th.setAttribute('data-sort-direction', ascending ? 'asc' : 'desc');
// Sort the results array by the given key
results.sort((a, b) => {
if (a[column] < b[column]) return ascending ? -1 : 1;
if (a[column] > b[column]) return ascending ? 1 : -1;
if (a[key] < b[key]) return ascending ? -1 : 1;
if (a[key] > b[key]) return ascending ? 1 : -1;
return 0;
});
// Clear the existing rows in tbody
tbody.innerHTML = '';
// Re-populate the table with sorted data
populateTableRows(tbody, results, columns, nest_level);
}
function loadResultsJSON(PR, sha, nameParams) {
function loadJSON(PR, sha, nameParams) {
const infoElement = document.getElementById('info-container');
let lastModifiedTime = null;
const task = nameParams[0].toLowerCase();
@ -663,9 +630,12 @@
let targetData = navigatePath(data, nameParams);
let nest_level = nameParams.length;
// Add footer links from top-level Result
if (Array.isArray(data.links) && data.links.length > 0) {
data.links.forEach(link => {
if (targetData) {
infoElement.style.display = 'none';
// Handle footer links if present
if (Array.isArray(data.aux_links) && data.aux_links.length > 0) {
data.aux_links.forEach(link => {
const a = document.createElement('a');
a.href = link;
a.textContent = link.split('/').pop();
@ -674,10 +644,6 @@
});
}
if (targetData) {
//infoElement.style.display = 'none';
infoElement.innerHTML = (targetData.info || '').replace(/\n/g, '<br>');
addStatusToStatus(targetData.status, targetData.start_time, targetData.duration)
// Handle links
@ -755,62 +721,22 @@
}
});
let path_commits_json = '';
let commitsArray = [];
if (PR) {
addKeyValueToStatus("PR", PR);
const baseUrl = window.location.origin + window.location.pathname.replace('/json.html', '');
path_commits_json = `${baseUrl}/${encodeURIComponent(PR)}/commits.json`;
addKeyValueToStatus("PR", PR)
} else {
// Placeholder for a different path when PR is missing
console.error("PR parameter is missing. Setting alternate commits path.");
path_commits_json = '/path/to/alternative/commits.json';
console.error("TODO")
}
function loadCommitsArray(path) {
return fetch(path, { cache: "no-cache" })
.then(response => {
if (!response.ok) {
console.error(`HTTP error! status: ${response.status}`)
return [];
}
return response.json();
})
.then(data => {
if (Array.isArray(data) && data.every(item => typeof item === 'object' && item.hasOwnProperty('sha'))) {
return data.map(item => item.sha);
} else {
throw new Error('Invalid data format: expected array of objects with a "sha" key');
}
})
.catch(error => {
console.error('Error loading commits JSON:', error);
return []; // Return an empty array if an error occurs
});
}
loadCommitsArray(path_commits_json)
.then(data => {
commitsArray = data;
})
.finally(() => {
// Proceed with the rest of the initialization
addKeyValueToStatus("sha", sha || "latest", commitsArray.concat(["latest"]));
addKeyValueToStatus("sha", sha);
if (nameParams[1]) {
addKeyValueToStatus("job", nameParams[1]);
}
addKeyValueToStatus("workflow", nameParams[0]);
// Check if all required parameters are present to load JSON
if (PR && sha && root_name) {
const shaToLoad = (sha === 'latest') ? commitsArray[commitsArray.length - 1] : sha;
loadResultsJSON(PR, shaToLoad, nameParams);
loadJSON(PR, sha, nameParams);
} else {
document.getElementById('title').textContent = 'Error: Missing required URL parameters: PR, sha, or name_0';
}
});
}
window.onload = init;

38
ci/praktika/mangle.py
View File

@ -1,10 +1,11 @@
import copy
import importlib.util
from pathlib import Path
from typing import Any, Dict
from praktika import Job
from praktika.settings import Settings
from praktika.utils import Utils
from praktika._settings import _USER_DEFINED_SETTINGS, _Settings
from praktika.utils import ContextManager, Utils
def _get_workflows(name=None, file=None):
@ -13,13 +14,14 @@ def _get_workflows(name=None, file=None):
"""
res = []
directory = Path(Settings.WORKFLOWS_DIRECTORY)
with ContextManager.cd():
directory = Path(_Settings.WORKFLOWS_DIRECTORY)
for py_file in directory.glob("*.py"):
if file and file not in str(py_file):
continue
module_name = py_file.name.removeprefix(".py")
spec = importlib.util.spec_from_file_location(
module_name, f"{Settings.WORKFLOWS_DIRECTORY}/{module_name}"
module_name, f"{_Settings.WORKFLOWS_DIRECTORY}/{module_name}"
)
assert spec
foo = importlib.util.module_from_spec(spec)
@ -56,6 +58,7 @@ def _update_workflow_artifacts(workflow):
artifact_job = {}
for job in workflow.jobs:
for artifact_name in job.provides:
assert artifact_name not in artifact_job
artifact_job[artifact_name] = job.name
for artifact in workflow.artifacts:
artifact._provided_by = artifact_job[artifact.name]
@ -105,3 +108,30 @@ def _update_workflow_with_native_jobs(workflow):
for job in workflow.jobs:
aux_job.requires.append(job.name)
workflow.jobs.append(aux_job)
def _get_user_settings() -> Dict[str, Any]:
"""
Gets user's settings
"""
res = {} # type: Dict[str, Any]
directory = Path(_Settings.SETTINGS_DIRECTORY)
for py_file in directory.glob("*.py"):
module_name = py_file.name.removeprefix(".py")
spec = importlib.util.spec_from_file_location(
module_name, f"{_Settings.SETTINGS_DIRECTORY}/{module_name}"
)
assert spec
foo = importlib.util.module_from_spec(spec)
assert spec.loader
spec.loader.exec_module(foo)
for setting in _USER_DEFINED_SETTINGS:
try:
value = getattr(foo, setting)
res[setting] = value
print(f"Apply user defined setting [{setting} = {value}]")
except Exception as e:
pass
return res

27
ci/praktika/native_jobs.py
View File

@ -10,8 +10,9 @@ from praktika.gh import GH
from praktika.hook_cache import CacheRunnerHooks
from praktika.hook_html import HtmlRunnerHooks
from praktika.mangle import _get_workflows
from praktika.result import Result, ResultInfo, _ResultS3
from praktika.result import Result, ResultInfo
from praktika.runtime import RunConfig
from praktika.s3 import S3
from praktika.settings import Settings
from praktika.utils import Shell, Utils
@ -150,7 +151,7 @@ def _config_workflow(workflow: Workflow.Config, job_name):
status = Result.Status.ERROR
print("ERROR: ", info)
else:
assert Shell.check(f"{Settings.PYTHON_INTERPRETER} -m praktika yaml")
Shell.check(f"{Settings.PYTHON_INTERPRETER} -m praktika --generate")
exit_code, output, err = Shell.get_res_stdout_stderr(
f"git diff-index HEAD -- {Settings.WORKFLOW_PATH_PREFIX}"
)
@ -224,7 +225,6 @@ def _config_workflow(workflow: Workflow.Config, job_name):
cache_success=[],
cache_success_base64=[],
cache_artifacts={},
cache_jobs={},
).dump()
# checks:
@ -250,9 +250,6 @@ def _config_workflow(workflow: Workflow.Config, job_name):
info_lines.append(job_name + ": " + info)
results.append(result_)
if workflow.enable_merge_commit:
assert False, "NOT implemented"
# config:
if workflow.dockers:
print("Calculate docker's digests")
@ -310,8 +307,9 @@ def _finish_workflow(workflow, job_name):
print(env.get_needs_statuses())
print("Check Workflow results")
_ResultS3.copy_result_from_s3(
S3.copy_result_from_s3(
Result.file_name_static(workflow.name),
lock=False,
)
workflow_result = Result.from_fs(workflow.name)
@ -341,12 +339,10 @@ def _finish_workflow(workflow, job_name):
f"NOTE: Result for [{result.name}] has not ok status [{result.status}]"
)
ready_for_merge_status = Result.Status.FAILED
failed_results.append(result.name)
failed_results.append(result.name.split("(", maxsplit=1)[0]) # cut name
if failed_results:
ready_for_merge_description = (
f'Failed {len(failed_results)} "Required for Merge" jobs'
)
ready_for_merge_description = f"failed: {', '.join(failed_results)}"
if not GH.post_commit_status(
name=Settings.READY_FOR_MERGE_STATUS_NAME + f" [{workflow.name}]",
@ -358,11 +354,14 @@ def _finish_workflow(workflow, job_name):
env.add_info(ResultInfo.GH_STATUS_ERROR)
if update_final_report:
_ResultS3.copy_result_to_s3(
S3.copy_result_to_s3(
workflow_result,
)
unlock=False,
) # no lock - no unlock
Result.from_fs(job_name).set_status(Result.Status.SUCCESS)
Result.from_fs(job_name).set_status(Result.Status.SUCCESS).set_info(
ready_for_merge_description
)
if __name__ == "__main__":

267
ci/praktika/result.py
View File

@ -1,13 +1,12 @@
import dataclasses
import datetime
import sys
from collections.abc import Container
from pathlib import Path
from typing import Any, Dict, List, Optional, Union
from typing import Any, Dict, List, Optional
from praktika._environment import _Environment
from praktika.cache import Cache
from praktika.s3 import S3
from praktika.settings import Settings
from praktika._settings import _Settings
from praktika.utils import ContextManager, MetaClasses, Shell, Utils
@ -28,6 +27,10 @@ class Result(MetaClasses.Serializable):
files (List[str]): A list of file paths or names related to the result.
links (List[str]): A list of URLs related to the result (e.g., links to reports or resources).
info (str): Additional information about the result. Free-form text.
# TODO: rename
aux_links (List[str]): A list of auxiliary links that provide additional context for the result.
# TODO: remove
html_link (str): A direct link to an HTML representation of the result (e.g., a detailed report page).
Inner Class:
Status: Defines possible statuses for the task, such as "success", "failure", etc.
@ -49,6 +52,8 @@ class Result(MetaClasses.Serializable):
files: List[str] = dataclasses.field(default_factory=list)
links: List[str] = dataclasses.field(default_factory=list)
info: str = ""
aux_links: List[str] = dataclasses.field(default_factory=list)
html_link: str = ""
@staticmethod
def create_from(
@ -57,15 +62,14 @@ class Result(MetaClasses.Serializable):
stopwatch: Utils.Stopwatch = None,
status="",
files=None,
info: Union[List[str], str] = "",
info="",
with_info_from_results=True,
):
if isinstance(status, bool):
status = Result.Status.SUCCESS if status else Result.Status.FAILED
if not results and not status:
Utils.raise_with_error(
f"Either .results ({results}) or .status ({status}) must be provided"
)
print("ERROR: Either .results or .status must be provided")
raise
if not name:
name = _Environment.get().JOB_NAME
if not name:
@ -74,10 +78,10 @@ class Result(MetaClasses.Serializable):
result_status = status or Result.Status.SUCCESS
infos = []
if info:
if isinstance(info, str):
infos += [info]
else:
if isinstance(info, Container):
infos += info
else:
infos.append(info)
if results and not status:
for result in results:
if result.status not in (Result.Status.SUCCESS, Result.Status.FAILED):
@ -108,7 +112,7 @@ class Result(MetaClasses.Serializable):
return self.status not in (Result.Status.PENDING, Result.Status.RUNNING)
def is_running(self):
return self.status in (Result.Status.RUNNING,)
return self.status not in (Result.Status.RUNNING,)
def is_ok(self):
return self.status in (Result.Status.SKIPPED, Result.Status.SUCCESS)
@ -151,7 +155,7 @@ class Result(MetaClasses.Serializable):
@classmethod
def file_name_static(cls, name):
return f"{Settings.TEMP_DIR}/result_{Utils.normalize_string(name)}.json"
return f"{_Settings.TEMP_DIR}/result_{Utils.normalize_string(name)}.json"
@classmethod
def from_dict(cls, obj: Dict[str, Any]) -> "Result":
@ -176,11 +180,6 @@ class Result(MetaClasses.Serializable):
)
return self
def set_timing(self, stopwatch: Utils.Stopwatch):
self.start_time = stopwatch.start_time
self.duration = stopwatch.duration
return self
def update_sub_result(self, result: "Result"):
assert self.results, "BUG?"
for i, result_ in enumerate(self.results):
@ -234,7 +233,7 @@ class Result(MetaClasses.Serializable):
)
@classmethod
def generate_skipped(cls, name, cache_record: Cache.CacheRecord, results=None):
def generate_skipped(cls, name, results=None):
return Result(
name=name,
status=Result.Status.SKIPPED,
@ -243,7 +242,7 @@ class Result(MetaClasses.Serializable):
results=results or [],
files=[],
links=[],
info=f"from cache: sha [{cache_record.sha}], pr/branch [{cache_record.pr_number or cache_record.branch}]",
info="from cache",
)
@classmethod
@ -277,7 +276,7 @@ class Result(MetaClasses.Serializable):
# Set log file path if logging is enabled
log_file = (
f"{Settings.TEMP_DIR}/{Utils.normalize_string(name)}.log"
f"{_Settings.TEMP_DIR}/{Utils.normalize_string(name)}.log"
if with_log
else None
)
@ -319,35 +318,18 @@ class Result(MetaClasses.Serializable):
files=[log_file] if log_file else None,
)
def complete_job(self):
def finish_job_accordingly(self):
self.dump()
if not self.is_ok():
print("ERROR: Job Failed")
print(self.to_stdout_formatted())
for result in self.results:
if not result.is_ok():
print("Failed checks:")
print(" | ", result)
sys.exit(1)
else:
print("ok")
def to_stdout_formatted(self, indent="", res=""):
if self.is_ok():
return res
res += f"{indent}Task [{self.name}] failed.\n"
fail_info = ""
sub_indent = indent + " "
if not self.results:
if not self.is_ok():
fail_info += f"{sub_indent}{self.name}:\n"
for line in self.info.splitlines():
fail_info += f"{sub_indent}{sub_indent}{line}\n"
return res + fail_info
for sub_result in self.results:
res = sub_result.to_stdout_formatted(sub_indent, res)
return res
class ResultInfo:
SETUP_ENV_JOB_FAILED = (
@ -370,202 +352,3 @@ class ResultInfo:
)
S3_ERROR = "S3 call failure"
class _ResultS3:
@classmethod
def copy_result_to_s3(cls, result, unlock=False):
result.dump()
env = _Environment.get()
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}"
s3_path_full = f"{s3_path}/{Path(result.file_name()).name}"
url = S3.copy_file_to_s3(s3_path=s3_path, local_path=result.file_name())
# if unlock:
# if not cls.unlock(s3_path_full):
# print(f"ERROR: File [{s3_path_full}] unlock failure")
# assert False # TODO: investigate
return url
@classmethod
def copy_result_from_s3(cls, local_path, lock=False):
env = _Environment.get()
file_name = Path(local_path).name
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/{file_name}"
# if lock:
# cls.lock(s3_path)
if not S3.copy_file_from_s3(s3_path=s3_path, local_path=local_path):
print(f"ERROR: failed to cp file [{s3_path}] from s3")
raise
@classmethod
def copy_result_from_s3_with_version(cls, local_path):
env = _Environment.get()
file_name = Path(local_path).name
local_dir = Path(local_path).parent
file_name_pattern = f"{file_name}_*"
for file_path in local_dir.glob(file_name_pattern):
file_path.unlink()
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/"
if not S3.copy_file_from_s3_matching_pattern(
s3_path=s3_path, local_path=local_dir, include=file_name_pattern
):
print(f"ERROR: failed to cp file [{s3_path}] from s3")
raise
result_files = []
for file_path in local_dir.glob(file_name_pattern):
result_files.append(file_path)
assert result_files, "No result files found"
result_files.sort()
version = int(result_files[-1].name.split("_")[-1])
Shell.check(f"cp {result_files[-1]} {local_path}", strict=True, verbose=True)
return version
@classmethod
def copy_result_to_s3_with_version(cls, result, version):
result.dump()
filename = Path(result.file_name()).name
file_name_versioned = f"{filename}_{str(version).zfill(3)}"
env = _Environment.get()
s3_path_versioned = (
f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/{file_name_versioned}"
)
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/"
if version == 0:
S3.clean_s3_directory(s3_path=s3_path)
if not S3.put(
s3_path=s3_path_versioned,
local_path=result.file_name(),
if_none_matched=True,
):
print("Failed to put versioned Result")
return False
if not S3.put(s3_path=s3_path, local_path=result.file_name()):
print("Failed to put non-versioned Result")
return True
# @classmethod
# def lock(cls, s3_path, level=0):
# env = _Environment.get()
# s3_path_lock = s3_path + f".lock"
# file_path_lock = f"{Settings.TEMP_DIR}/{Path(s3_path_lock).name}"
# assert Shell.check(
# f"echo '''{env.JOB_NAME}''' > {file_path_lock}", verbose=True
# ), "Never"
#
# i = 20
# meta = S3.head_object(s3_path_lock)
# while meta:
# locked_by_job = meta.get("Metadata", {"job": ""}).get("job", "")
# if locked_by_job:
# decoded_bytes = base64.b64decode(locked_by_job)
# locked_by_job = decoded_bytes.decode("utf-8")
# print(
# f"WARNING: Failed to acquire lock, meta [{meta}], job [{locked_by_job}] - wait"
# )
# i -= 5
# if i < 0:
# info = f"ERROR: lock acquire failure - unlock forcefully"
# print(info)
# env.add_info(info)
# break
# time.sleep(5)
#
# metadata = {"job": Utils.to_base64(env.JOB_NAME)}
# S3.put(
# s3_path=s3_path_lock,
# local_path=file_path_lock,
# metadata=metadata,
# if_none_matched=True,
# )
# time.sleep(1)
# obj = S3.head_object(s3_path_lock)
# if not obj or not obj.has_tags(tags=metadata):
# print(f"WARNING: locked by another job [{obj}]")
# env.add_info("S3 lock file failure")
# cls.lock(s3_path, level=level + 1)
# print("INFO: lock acquired")
#
# @classmethod
# def unlock(cls, s3_path):
# s3_path_lock = s3_path + ".lock"
# env = _Environment.get()
# obj = S3.head_object(s3_path_lock)
# if not obj:
# print("ERROR: lock file is removed")
# assert False # investigate
# elif not obj.has_tags({"job": Utils.to_base64(env.JOB_NAME)}):
# print("ERROR: lock file was acquired by another job")
# assert False # investigate
#
# if not S3.delete(s3_path_lock):
# print(f"ERROR: File [{s3_path_lock}] delete failure")
# print("INFO: lock released")
# return True
@classmethod
def upload_result_files_to_s3(cls, result):
if result.results:
for result_ in result.results:
cls.upload_result_files_to_s3(result_)
for file in result.files:
if not Path(file).is_file():
print(f"ERROR: Invalid file [{file}] in [{result.name}] - skip upload")
result.info += f"\nWARNING: Result file [{file}] was not found"
file_link = S3._upload_file_to_s3(file, upload_to_s3=False)
else:
is_text = False
for text_file_suffix in Settings.TEXT_CONTENT_EXTENSIONS:
if file.endswith(text_file_suffix):
print(
f"File [{file}] matches Settings.TEXT_CONTENT_EXTENSIONS [{Settings.TEXT_CONTENT_EXTENSIONS}] - add text attribute for s3 object"
)
is_text = True
break
file_link = S3._upload_file_to_s3(
file,
upload_to_s3=True,
text=is_text,
s3_subprefix=Utils.normalize_string(result.name),
)
result.links.append(file_link)
if result.files:
print(
f"Result files [{result.files}] uploaded to s3 [{result.links[-len(result.files):]}] - clean files list"
)
result.files = []
result.dump()
@classmethod
def update_workflow_results(cls, workflow_name, new_info="", new_sub_results=None):
assert new_info or new_sub_results
attempt = 1
prev_status = ""
new_status = ""
done = False
while attempt < 10:
version = cls.copy_result_from_s3_with_version(
Result.file_name_static(workflow_name)
)
workflow_result = Result.from_fs(workflow_name)
prev_status = workflow_result.status
if new_info:
workflow_result.set_info(new_info)
if new_sub_results:
if isinstance(new_sub_results, Result):
new_sub_results = [new_sub_results]
for result_ in new_sub_results:
workflow_result.update_sub_result(result_)
new_status = workflow_result.status
if cls.copy_result_to_s3_with_version(workflow_result, version=version + 1):
done = True
break
print(f"Attempt [{attempt}] to upload workflow result failed")
attempt += 1
assert done
if prev_status != new_status:
return new_status
else:
return None

87
ci/praktika/runner.py
View File

@ -19,7 +19,7 @@ from praktika.utils import Shell, TeePopen, Utils
class Runner:
@staticmethod
def generate_local_run_environment(workflow, job, pr=None, branch=None, sha=None):
def generate_dummy_environment(workflow, job):
print("WARNING: Generate dummy env for local test")
Shell.check(
f"mkdir -p {Settings.TEMP_DIR} {Settings.INPUT_DIR} {Settings.OUTPUT_DIR}"
@ -28,9 +28,9 @@ class Runner:
WORKFLOW_NAME=workflow.name,
JOB_NAME=job.name,
REPOSITORY="",
BRANCH=branch or Settings.MAIN_BRANCH if not pr else "",
SHA=sha or Shell.get_output("git rev-parse HEAD"),
PR_NUMBER=pr or -1,
BRANCH="",
SHA="",
PR_NUMBER=-1,
EVENT_TYPE="",
JOB_OUTPUT_STREAM="",
EVENT_FILE_PATH="",
@ -52,7 +52,6 @@ class Runner:
cache_success=[],
cache_success_base64=[],
cache_artifacts={},
cache_jobs={},
)
for docker in workflow.dockers:
workflow_config.digest_dockers[docker.name] = Digest().calc_docker_digest(
@ -81,12 +80,13 @@ class Runner:
print("Read GH Environment")
env = _Environment.from_env()
env.JOB_NAME = job.name
env.PARAMETER = job.parameter
env.dump()
print(env)
return 0
def _pre_run(self, workflow, job, local_run=False):
def _pre_run(self, workflow, job):
env = _Environment.get()
result = Result(
@ -96,7 +96,6 @@ class Runner:
)
result.dump()
if not local_run:
if workflow.enable_report and job.name != Settings.CI_CONFIG_JOB_NAME:
print("Update Job and Workflow Report")
HtmlRunnerHooks.pre_run(workflow, job)
@ -124,48 +123,28 @@ class Runner:
return 0
def _run(self, workflow, job, docker="", no_docker=False, param=None, test=""):
# re-set envs for local run
env = _Environment.get()
env.JOB_NAME = job.name
env.dump()
def _run(self, workflow, job, docker="", no_docker=False, param=None):
if param:
if not isinstance(param, str):
Utils.raise_with_error(
f"Custom param for local tests must be of type str, got [{type(param)}]"
)
env = _Environment.get()
env.dump()
if job.run_in_docker and not no_docker:
job.run_in_docker, docker_settings = (
job.run_in_docker.split("+")[0],
job.run_in_docker.split("+")[1:],
)
from_root = "root" in docker_settings
settings = [s for s in docker_settings if s.startswith("--")]
if ":" in job.run_in_docker:
docker_name, docker_tag = job.run_in_docker.split(":")
print(
f"WARNING: Job [{job.name}] use custom docker image with a tag - praktika won't control docker version"
)
else:
docker_name, docker_tag = (
job.run_in_docker,
RunConfig.from_fs(workflow.name).digest_dockers[job.run_in_docker],
)
docker = docker or f"{docker_name}:{docker_tag}"
cmd = f"docker run --rm --name praktika {'--user $(id -u):$(id -g)' if not from_root else ''} -e PYTHONPATH='{Settings.DOCKER_WD}:{Settings.DOCKER_WD}/ci' --volume ./:{Settings.DOCKER_WD} --volume {Settings.TEMP_DIR}:{Settings.TEMP_DIR} --workdir={Settings.DOCKER_WD} {' '.join(settings)} {docker} {job.command}"
# TODO: add support for any image, including not from ci config (e.g. ubuntu:latest)
docker_tag = RunConfig.from_fs(workflow.name).digest_dockers[
job.run_in_docker
]
docker = docker or f"{job.run_in_docker}:{docker_tag}"
cmd = f"docker run --rm --user \"$(id -u):$(id -g)\" -e PYTHONPATH='{Settings.DOCKER_WD}:{Settings.DOCKER_WD}/ci' --volume ./:{Settings.DOCKER_WD} --volume {Settings.TEMP_DIR}:{Settings.TEMP_DIR} --workdir={Settings.DOCKER_WD} {docker} {job.command}"
else:
cmd = job.command
python_path = os.getenv("PYTHONPATH", ":")
os.environ["PYTHONPATH"] = f".:{python_path}"
if param:
print(f"Custom --param [{param}] will be passed to job's script")
cmd += f" --param {param}"
if test:
print(f"Custom --test [{test}] will be passed to job's script")
cmd += f" --test {test}"
print(f"--- Run command [{cmd}]")
with TeePopen(cmd, timeout=job.timeout) as process:
@ -240,10 +219,13 @@ class Runner:
print(info)
result.set_info(info).set_status(Result.Status.ERROR).dump()
if not result.is_ok():
result.set_files(files=[Settings.RUN_LOG])
result.update_duration().dump()
if result.info and result.status != Result.Status.SUCCESS:
# provide job info to workflow level
info_errors.append(result.info)
if run_exit_code == 0:
providing_artifacts = []
if job.provides and workflow.artifacts:
@ -303,24 +285,14 @@ class Runner:
return True
def run(
self,
workflow,
job,
docker="",
local_run=False,
no_docker=False,
param=None,
test="",
pr=None,
sha=None,
branch=None,
self, workflow, job, docker="", dummy_env=False, no_docker=False, param=None
):
res = True
setup_env_code = -10
prerun_code = -10
run_code = -10
if res and not local_run:
if res and not dummy_env:
print(
f"\n\n=== Setup env script [{job.name}], workflow [{workflow.name}] ==="
)
@ -337,15 +309,13 @@ class Runner:
traceback.print_exc()
print(f"=== Setup env finished ===\n\n")
else:
self.generate_local_run_environment(
workflow, job, pr=pr, branch=branch, sha=sha
)
self.generate_dummy_environment(workflow, job)
if res and (not local_run or pr or sha or branch):
if res and not dummy_env:
res = False
print(f"=== Pre run script [{job.name}], workflow [{workflow.name}] ===")
try:
prerun_code = self._pre_run(workflow, job, local_run=local_run)
prerun_code = self._pre_run(workflow, job)
res = prerun_code == 0
if not res:
print(f"ERROR: Pre-run failed with exit code [{prerun_code}]")
@ -359,12 +329,7 @@ class Runner:
print(f"=== Run script [{job.name}], workflow [{workflow.name}] ===")
try:
run_code = self._run(
workflow,
job,
docker=docker,
no_docker=no_docker,
param=param,
test=test,
workflow, job, docker=docker, no_docker=no_docker, param=param
)
res = run_code == 0
if not res:
@ -374,7 +339,7 @@ class Runner:
traceback.print_exc()
print(f"=== Run scrip finished ===\n\n")
if not local_run:
if not dummy_env:
print(f"=== Post run script [{job.name}], workflow [{workflow.name}] ===")
self._post_run(workflow, job, setup_env_code, prerun_code, run_code)
print(f"=== Post run scrip finished ===")

6
ci/praktika/runtime.py
View File

@ -15,23 +15,17 @@ class RunConfig(MetaClasses.Serializable):
# there are might be issue with special characters in job names if used directly in yaml syntax - create base64 encoded list to avoid this
cache_success_base64: List[str]
cache_artifacts: Dict[str, Cache.CacheRecord]
cache_jobs: Dict[str, Cache.CacheRecord]
sha: str
@classmethod
def from_dict(cls, obj):
cache_artifacts = obj["cache_artifacts"]
cache_jobs = obj["cache_jobs"]
cache_artifacts_deserialized = {}
cache_jobs_deserialized = {}
for artifact_name, cache_artifact in cache_artifacts.items():
cache_artifacts_deserialized[artifact_name] = Cache.CacheRecord.from_dict(
cache_artifact
)
obj["cache_artifacts"] = cache_artifacts_deserialized
for job_name, cache_jobs in cache_jobs.items():
cache_jobs_deserialized[job_name] = Cache.CacheRecord.from_dict(cache_jobs)
obj["cache_jobs"] = cache_artifacts_deserialized
return RunConfig(**obj)
@classmethod

174
ci/praktika/s3.py
View File

@ -1,11 +1,12 @@
import dataclasses
import json
import time
from pathlib import Path
from typing import Dict
from praktika._environment import _Environment
from praktika.settings import Settings
from praktika.utils import Shell
from praktika.utils import Shell, Utils
class S3:
@ -51,22 +52,23 @@ class S3:
cmd += " --content-type text/plain"
res = cls.run_command_with_retries(cmd)
if not res:
raise RuntimeError()
raise
bucket = s3_path.split("/")[0]
endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket]
assert endpoint
return f"https://{s3_full_path}".replace(bucket, endpoint)
@classmethod
def put(cls, s3_path, local_path, text=False, metadata=None, if_none_matched=False):
def put(cls, s3_path, local_path, text=False, metadata=None):
assert Path(local_path).exists(), f"Path [{local_path}] does not exist"
assert Path(s3_path), f"Invalid S3 Path [{s3_path}]"
assert Path(
local_path
).is_file(), f"Path [{local_path}] is not file. Only files are supported"
file_name = Path(local_path).name
s3_full_path = s3_path
if s3_full_path.endswith("/"):
s3_full_path = f"{s3_path}{Path(local_path).name}"
if not s3_full_path.endswith(file_name):
s3_full_path = f"{s3_path}/{Path(local_path).name}"
s3_full_path = str(s3_full_path).removeprefix("s3://")
bucket, key = s3_full_path.split("/", maxsplit=1)
@ -74,8 +76,6 @@ class S3:
command = (
f"aws s3api put-object --bucket {bucket} --key {key} --body {local_path}"
)
if if_none_matched:
command += f' --if-none-match "*"'
if metadata:
for k, v in metadata.items():
command += f" --metadata {k}={v}"
@ -84,7 +84,7 @@ class S3:
if text:
cmd += " --content-type text/plain"
res = cls.run_command_with_retries(command)
return res
assert res
@classmethod
def run_command_with_retries(cls, command, retries=Settings.MAX_RETRIES_S3):
@ -101,14 +101,6 @@ class S3:
elif "does not exist" in stderr:
print("ERROR: requested file does not exist")
break
elif "Unknown options" in stderr:
print("ERROR: Invalid AWS CLI command or CLI client version:")
print(f" | awc error: {stderr}")
break
elif "PreconditionFailed" in stderr:
print("ERROR: AWS API Call Precondition Failed")
print(f" | awc error: {stderr}")
break
if ret_code != 0:
print(
f"ERROR: aws s3 cp failed, stdout/stderr err: [{stderr}], out [{stdout}]"
@ -116,6 +108,13 @@ class S3:
res = ret_code == 0
return res
@classmethod
def get_link(cls, s3_path, local_path):
s3_full_path = f"{s3_path}/{Path(local_path).name}"
bucket = s3_path.split("/")[0]
endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket]
return f"https://{s3_full_path}".replace(bucket, endpoint)
@classmethod
def copy_file_from_s3(cls, s3_path, local_path):
assert Path(s3_path), f"Invalid S3 Path [{s3_path}]"
@ -129,19 +128,6 @@ class S3:
res = cls.run_command_with_retries(cmd)
return res
@classmethod
def copy_file_from_s3_matching_pattern(
cls, s3_path, local_path, include, exclude="*"
):
assert Path(s3_path), f"Invalid S3 Path [{s3_path}]"
assert Path(
local_path
).is_dir(), f"Path [{local_path}] does not exist or not a directory"
assert s3_path.endswith("/"), f"s3 path is invalid [{s3_path}]"
cmd = f'aws s3 cp s3://{s3_path} {local_path} --exclude "{exclude}" --include "{include}" --recursive'
res = cls.run_command_with_retries(cmd)
return res
@classmethod
def head_object(cls, s3_path):
s3_path = str(s3_path).removeprefix("s3://")
@ -162,6 +148,103 @@ class S3:
verbose=True,
)
# TODO: apparently should be placed into separate file to be used only inside praktika
# keeping this module clean from importing Settings, Environment and etc, making it easy for use externally
@classmethod
def copy_result_to_s3(cls, result, unlock=True):
result.dump()
env = _Environment.get()
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}"
s3_path_full = f"{s3_path}/{Path(result.file_name()).name}"
url = S3.copy_file_to_s3(s3_path=s3_path, local_path=result.file_name())
if env.PR_NUMBER:
print("Duplicate Result for latest commit alias in PR")
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix(latest=True)}"
url = S3.copy_file_to_s3(s3_path=s3_path, local_path=result.file_name())
if unlock:
if not cls.unlock(s3_path_full):
print(f"ERROR: File [{s3_path_full}] unlock failure")
assert False # TODO: investigate
return url
@classmethod
def copy_result_from_s3(cls, local_path, lock=True):
env = _Environment.get()
file_name = Path(local_path).name
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}/{file_name}"
if lock:
cls.lock(s3_path)
if not S3.copy_file_from_s3(s3_path=s3_path, local_path=local_path):
print(f"ERROR: failed to cp file [{s3_path}] from s3")
raise
@classmethod
def lock(cls, s3_path, level=0):
assert level < 3, "Never"
env = _Environment.get()
s3_path_lock = s3_path + f".lock"
file_path_lock = f"{Settings.TEMP_DIR}/{Path(s3_path_lock).name}"
assert Shell.check(
f"echo '''{env.JOB_NAME}''' > {file_path_lock}", verbose=True
), "Never"
i = 20
meta = S3.head_object(s3_path_lock)
while meta:
print(f"WARNING: Failed to acquire lock, meta [{meta}] - wait")
i -= 5
if i < 0:
info = f"ERROR: lock acquire failure - unlock forcefully"
print(info)
env.add_info(info)
break
time.sleep(5)
metadata = {"job": Utils.to_base64(env.JOB_NAME)}
S3.put(
s3_path=s3_path_lock,
local_path=file_path_lock,
metadata=metadata,
)
time.sleep(1)
obj = S3.head_object(s3_path_lock)
if not obj or not obj.has_tags(tags=metadata):
print(f"WARNING: locked by another job [{obj}]")
env.add_info("S3 lock file failure")
cls.lock(s3_path, level=level + 1)
print("INFO: lock acquired")
@classmethod
def unlock(cls, s3_path):
s3_path_lock = s3_path + ".lock"
env = _Environment.get()
obj = S3.head_object(s3_path_lock)
if not obj:
print("ERROR: lock file is removed")
assert False # investigate
elif not obj.has_tags({"job": Utils.to_base64(env.JOB_NAME)}):
print("ERROR: lock file was acquired by another job")
assert False # investigate
if not S3.delete(s3_path_lock):
print(f"ERROR: File [{s3_path_lock}] delete failure")
print("INFO: lock released")
return True
@classmethod
def get_result_link(cls, result):
env = _Environment.get()
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix(latest=True if env.PR_NUMBER else False)}"
return S3.get_link(s3_path=s3_path, local_path=result.file_name())
@classmethod
def clean_latest_result(cls):
env = _Environment.get()
env.SHA = "latest"
assert env.PR_NUMBER
s3_path = f"{Settings.HTML_S3_PATH}/{env.get_s3_prefix()}"
S3.clean_s3_directory(s3_path=s3_path)
@classmethod
def _upload_file_to_s3(
cls, local_file_path, upload_to_s3: bool, text: bool = False, s3_subprefix=""
@ -177,3 +260,36 @@ class S3:
)
return html_link
return f"file://{Path(local_file_path).absolute()}"
@classmethod
def upload_result_files_to_s3(cls, result):
if result.results:
for result_ in result.results:
cls.upload_result_files_to_s3(result_)
for file in result.files:
if not Path(file).is_file():
print(f"ERROR: Invalid file [{file}] in [{result.name}] - skip upload")
result.info += f"\nWARNING: Result file [{file}] was not found"
file_link = cls._upload_file_to_s3(file, upload_to_s3=False)
else:
is_text = False
for text_file_suffix in Settings.TEXT_CONTENT_EXTENSIONS:
if file.endswith(text_file_suffix):
print(
f"File [{file}] matches Settings.TEXT_CONTENT_EXTENSIONS [{Settings.TEXT_CONTENT_EXTENSIONS}] - add text attribute for s3 object"
)
is_text = True
break
file_link = cls._upload_file_to_s3(
file,
upload_to_s3=True,
text=is_text,
s3_subprefix=Utils.normalize_string(result.name),
)
result.links.append(file_link)
if result.files:
print(
f"Result files [{result.files}] uploaded to s3 [{result.links[-len(result.files):]}] - clean files list"
)
result.files = []
result.dump()

156
ci/praktika/settings.py
View File

@ -1,152 +1,8 @@
import dataclasses
import importlib.util
from pathlib import Path
from typing import Dict, Iterable, List, Optional
from praktika._settings import _Settings
from praktika.mangle import _get_user_settings
Settings = _Settings()
@dataclasses.dataclass
class _Settings:
######################################
# Pipeline generation settings #
######################################
MAIN_BRANCH = "main"
CI_PATH = "./ci"
WORKFLOW_PATH_PREFIX: str = "./.github/workflows"
WORKFLOWS_DIRECTORY: str = f"{CI_PATH}/workflows"
SETTINGS_DIRECTORY: str = f"{CI_PATH}/settings"
CI_CONFIG_JOB_NAME = "Config Workflow"
DOCKER_BUILD_JOB_NAME = "Docker Builds"
FINISH_WORKFLOW_JOB_NAME = "Finish Workflow"
READY_FOR_MERGE_STATUS_NAME = "Ready for Merge"
CI_CONFIG_RUNS_ON: Optional[List[str]] = None
DOCKER_BUILD_RUNS_ON: Optional[List[str]] = None
VALIDATE_FILE_PATHS: bool = True
######################################
# Runtime Settings #
######################################
MAX_RETRIES_S3 = 3
MAX_RETRIES_GH = 3
######################################
# S3 (artifact storage) settings #
######################################
S3_ARTIFACT_PATH: str = ""
######################################
# CI workspace settings #
######################################
TEMP_DIR: str = "/tmp/praktika"
OUTPUT_DIR: str = f"{TEMP_DIR}/output"
INPUT_DIR: str = f"{TEMP_DIR}/input"
PYTHON_INTERPRETER: str = "python3"
PYTHON_PACKET_MANAGER: str = "pip3"
PYTHON_VERSION: str = "3.9"
INSTALL_PYTHON_FOR_NATIVE_JOBS: bool = False
INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS: str = "./ci/requirements.txt"
ENVIRONMENT_VAR_FILE: str = f"{TEMP_DIR}/environment.json"
RUN_LOG: str = f"{TEMP_DIR}/praktika_run.log"
SECRET_GH_APP_ID: str = "GH_APP_ID"
SECRET_GH_APP_PEM_KEY: str = "GH_APP_PEM_KEY"
ENV_SETUP_SCRIPT: str = "/tmp/praktika_setup_env.sh"
WORKFLOW_STATUS_FILE: str = f"{TEMP_DIR}/workflow_status.json"
######################################
# CI Cache settings #
######################################
CACHE_VERSION: int = 1
CACHE_DIGEST_LEN: int = 20
CACHE_S3_PATH: str = ""
CACHE_LOCAL_PATH: str = f"{TEMP_DIR}/ci_cache"
######################################
# Report settings #
######################################
HTML_S3_PATH: str = ""
HTML_PAGE_FILE: str = "./praktika/json.html"
TEXT_CONTENT_EXTENSIONS: Iterable[str] = frozenset([".txt", ".log"])
S3_BUCKET_TO_HTTP_ENDPOINT: Optional[Dict[str, str]] = None
DOCKERHUB_USERNAME: str = ""
DOCKERHUB_SECRET: str = ""
DOCKER_WD: str = "/wd"
######################################
# CI DB Settings #
######################################
SECRET_CI_DB_URL: str = "CI_DB_URL"
SECRET_CI_DB_PASSWORD: str = "CI_DB_PASSWORD"
CI_DB_DB_NAME = ""
CI_DB_TABLE_NAME = ""
CI_DB_INSERT_TIMEOUT_SEC = 5
DISABLE_MERGE_COMMIT = True
_USER_DEFINED_SETTINGS = [
"S3_ARTIFACT_PATH",
"CACHE_S3_PATH",
"HTML_S3_PATH",
"S3_BUCKET_TO_HTTP_ENDPOINT",
"TEXT_CONTENT_EXTENSIONS",
"TEMP_DIR",
"OUTPUT_DIR",
"INPUT_DIR",
"CI_CONFIG_RUNS_ON",
"DOCKER_BUILD_RUNS_ON",
"CI_CONFIG_JOB_NAME",
"PYTHON_INTERPRETER",
"PYTHON_VERSION",
"PYTHON_PACKET_MANAGER",
"INSTALL_PYTHON_FOR_NATIVE_JOBS",
"INSTALL_PYTHON_REQS_FOR_NATIVE_JOBS",
"MAX_RETRIES_S3",
"MAX_RETRIES_GH",
"VALIDATE_FILE_PATHS",
"DOCKERHUB_USERNAME",
"DOCKERHUB_SECRET",
"READY_FOR_MERGE_STATUS_NAME",
"SECRET_CI_DB_URL",
"SECRET_CI_DB_PASSWORD",
"CI_DB_DB_NAME",
"CI_DB_TABLE_NAME",
"CI_DB_INSERT_TIMEOUT_SEC",
"SECRET_GH_APP_PEM_KEY",
"SECRET_GH_APP_ID",
"MAIN_BRANCH",
"DISABLE_MERGE_COMMIT",
]
def _get_settings() -> _Settings:
res = _Settings()
directory = Path(_Settings.SETTINGS_DIRECTORY)
for py_file in directory.glob("*.py"):
module_name = py_file.name.removeprefix(".py")
spec = importlib.util.spec_from_file_location(
module_name, f"{_Settings.SETTINGS_DIRECTORY}/{module_name}"
)
assert spec
foo = importlib.util.module_from_spec(spec)
assert spec.loader
spec.loader.exec_module(foo)
for setting in _USER_DEFINED_SETTINGS:
try:
value = getattr(foo, setting)
res.__setattr__(setting, value)
# print(f"- read user defined setting [{setting} = {value}]")
except Exception as e:
# print(f"Exception while read user settings: {e}")
pass
return res
class GHRunners:
ubuntu = "ubuntu-latest"
Settings = _get_settings()
user_settings = _get_user_settings()
for setting, value in user_settings.items():
Settings.__setattr__(setting, value)

27
ci/praktika/utils.py
View File

@ -17,6 +17,8 @@ from threading import Thread
from types import SimpleNamespace
from typing import Any, Dict, Iterator, List, Optional, Type, TypeVar, Union
from praktika._settings import _Settings
T = TypeVar("T", bound="Serializable")
@ -79,25 +81,24 @@ class MetaClasses:
class ContextManager:
@staticmethod
@contextmanager
def cd(to: Optional[Union[Path, str]]) -> Iterator[None]:
def cd(to: Optional[Union[Path, str]] = None) -> Iterator[None]:
"""
changes current working directory to @path or `git root` if @path is None
:param to:
:return:
"""
# if not to:
# try:
# to = Shell.get_output_or_raise("git rev-parse --show-toplevel")
# except:
# pass
# if not to:
# if Path(_Settings.DOCKER_WD).is_dir():
# to = _Settings.DOCKER_WD
# if not to:
# assert False, "FIX IT"
# assert to
if not to:
try:
to = Shell.get_output_or_raise("git rev-parse --show-toplevel")
except:
pass
if not to:
if Path(_Settings.DOCKER_WD).is_dir():
to = _Settings.DOCKER_WD
if not to:
assert False, "FIX IT"
assert to
old_pwd = os.getcwd()
if to:
os.chdir(to)
try:
yield

11
ci/praktika/validator.py
View File

@ -4,8 +4,10 @@ from itertools import chain
from pathlib import Path
from praktika import Workflow
from praktika._settings import GHRunners
from praktika.mangle import _get_workflows
from praktika.settings import GHRunners, Settings
from praktika.settings import Settings
from praktika.utils import ContextManager
class Validator:
@ -117,6 +119,7 @@ class Validator:
def validate_file_paths_in_run_command(cls, workflow: Workflow.Config) -> None:
if not Settings.VALIDATE_FILE_PATHS:
return
with ContextManager.cd():
for job in workflow.jobs:
run_command = job.command
command_parts = run_command.split(" ")
@ -132,6 +135,7 @@ class Validator:
def validate_file_paths_in_digest_configs(cls, workflow: Workflow.Config) -> None:
if not Settings.VALIDATE_FILE_PATHS:
return
with ContextManager.cd():
for job in workflow.jobs:
if not job.digest_config:
continue
@ -149,6 +153,7 @@ class Validator:
@classmethod
def validate_requirements_txt_files(cls, workflow: Workflow.Config) -> None:
with ContextManager.cd():
for job in workflow.jobs:
if job.job_requirements:
if job.job_requirements.python_requirements_txt:
@ -166,7 +171,9 @@ class Validator:
"\n echo requests==2.32.3 >> ./ci/requirements.txt"
)
message += "\n echo https://clickhouse-builds.s3.amazonaws.com/packages/praktika-0.1-py3-none-any.whl >> ./ci/requirements.txt"
cls.evaluate_check(path.is_file(), message, job.name, workflow.name)
cls.evaluate_check(
path.is_file(), message, job.name, workflow.name
)
@classmethod
def validate_dockers(cls, workflow: Workflow.Config):

1
ci/praktika/workflow.py
View File

@ -31,7 +31,6 @@ class Workflow:
enable_report: bool = False
enable_merge_ready_status: bool = False
enable_cidb: bool = False
enable_merge_commit: bool = False
def is_event_pull_request(self):
return self.event == Workflow.Event.PULL_REQUEST

10
ci/praktika/yaml_generator.py
View File

@ -80,8 +80,6 @@ jobs:
steps:
- name: Checkout code
uses: actions/checkout@v4
with:
ref: ${{{{ github.head_ref }}}}
{JOB_ADDONS}
- name: Prepare env script
run: |
@ -104,11 +102,7 @@ jobs:
run: |
. /tmp/praktika_setup_env.sh
set -o pipefail
if command -v ts &> /dev/null; then
python3 -m praktika run --job '''{JOB_NAME}''' --workflow "{WORKFLOW_NAME}" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee /tmp/praktika/praktika_run.log
else
python3 -m praktika run --job '''{JOB_NAME}''' --workflow "{WORKFLOW_NAME}" --ci |& tee /tmp/praktika/praktika_run.log
fi
{PYTHON} -m praktika run --job '''{JOB_NAME}''' --workflow "{WORKFLOW_NAME}" --ci |& tee {RUN_LOG}
{UPLOADS_GITHUB}\
"""
@ -190,9 +184,11 @@ jobs:
False
), f"Workflow event not yet supported [{workflow_config.event}]"
with ContextManager.cd():
with open(self._get_workflow_file_name(workflow_config.name), "w") as f:
f.write(yaml_workflow_str)
with ContextManager.cd():
Shell.check("git add ./.github/workflows/*.yaml")

55
ci/settings/definitions.py
View File

@ -7,33 +7,24 @@ S3_BUCKET_HTTP_ENDPOINT = "clickhouse-builds.s3.amazonaws.com"
class RunnerLabels:
CI_SERVICES = "ci_services"
CI_SERVICES_EBS = "ci_services_ebs"
BUILDER_AMD = "builder"
BUILDER_ARM = "builder-aarch64"
FUNC_TESTER_AMD = "func-tester"
FUNC_TESTER_ARM = "func-tester-aarch64"
BUILDER = "builder"
BASE_BRANCH = "master"
azure_secret = Secret.Config(
name="azure_connection_string",
type=Secret.Type.AWS_SSM_VAR,
)
SECRETS = [
Secret.Config(
name="dockerhub_robot_password",
type=Secret.Type.AWS_SSM_VAR,
),
azure_secret,
# Secret.Config(
# name="woolenwolf_gh_app.clickhouse-app-id",
# type=Secret.Type.AWS_SSM_SECRET,
# ),
# Secret.Config(
# name="woolenwolf_gh_app.clickhouse-app-key",
# type=Secret.Type.AWS_SSM_SECRET,
# ),
Secret.Config(
name="woolenwolf_gh_app.clickhouse-app-id",
type=Secret.Type.AWS_SSM_SECRET,
),
Secret.Config(
name="woolenwolf_gh_app.clickhouse-app-key",
type=Secret.Type.AWS_SSM_SECRET,
),
]
DOCKERS = [
@ -127,18 +118,18 @@ DOCKERS = [
# platforms=Docker.Platforms.arm_amd,
# depends_on=["clickhouse/test-base"],
# ),
Docker.Config(
name="clickhouse/stateless-test",
path="./ci/docker/stateless-test",
platforms=Docker.Platforms.arm_amd,
depends_on=[],
),
Docker.Config(
name="clickhouse/stateful-test",
path="./ci/docker/stateful-test",
platforms=Docker.Platforms.arm_amd,
depends_on=["clickhouse/stateless-test"],
),
# Docker.Config(
# name="clickhouse/stateless-test",
# path="./ci/docker/test/stateless",
# platforms=Docker.Platforms.arm_amd,
# depends_on=["clickhouse/test-base"],
# ),
# Docker.Config(
# name="clickhouse/stateful-test",
# path="./ci/docker/test/stateful",
# platforms=Docker.Platforms.arm_amd,
# depends_on=["clickhouse/stateless-test"],
# ),
# Docker.Config(
# name="clickhouse/stress-test",
# path="./ci/docker/test/stress",
@ -239,6 +230,4 @@ DOCKERS = [
class JobNames:
STYLE_CHECK = "Style Check"
FAST_TEST = "Fast test"
BUILD = "Build"
STATELESS = "Stateless tests"
STATEFUL = "Stateful tests"
BUILD_AMD_DEBUG = "Build amd64 debug"

2
ci/settings/settings.py
View File

@ -4,8 +4,6 @@ from ci.settings.definitions import (
RunnerLabels,
)
MAIN_BRANCH = "master"
S3_ARTIFACT_PATH = f"{S3_BUCKET_NAME}/artifacts"
CI_CONFIG_RUNS_ON = [RunnerLabels.CI_SERVICES]
DOCKER_BUILD_RUNS_ON = [RunnerLabels.CI_SERVICES_EBS]

135
ci/workflows/pull_request.py
View File

@ -1,3 +1,5 @@
from typing import List
from praktika import Artifact, Job, Workflow
from praktika.settings import Settings
@ -11,10 +13,7 @@ from ci.settings.definitions import (
class ArtifactNames:
CH_AMD_DEBUG = "CH_AMD_DEBUG"
CH_AMD_RELEASE = "CH_AMD_RELEASE"
CH_ARM_RELEASE = "CH_ARM_RELEASE"
CH_ARM_ASAN = "CH_ARM_ASAN"
ch_debug_binary = "clickhouse_debug_binary"
style_check_job = Job.Config(
@ -26,7 +25,7 @@ style_check_job = Job.Config(
fast_test_job = Job.Config(
name=JobNames.FAST_TEST,
runs_on=[RunnerLabels.BUILDER_AMD],
runs_on=[RunnerLabels.BUILDER],
command="python3 ./ci/jobs/fast_test.py",
run_in_docker="clickhouse/fasttest",
digest_config=Job.CacheDigestConfig(
@ -38,13 +37,11 @@ fast_test_job = Job.Config(
),
)
build_jobs = Job.Config(
name=JobNames.BUILD,
runs_on=["...from params..."],
requires=[JobNames.FAST_TEST],
command="python3 ./ci/jobs/build_clickhouse.py --build-type {PARAMETER}",
job_build_amd_debug = Job.Config(
name=JobNames.BUILD_AMD_DEBUG,
runs_on=[RunnerLabels.BUILDER],
command="python3 ./ci/jobs/build_clickhouse.py amd_debug",
run_in_docker="clickhouse/fasttest",
timeout=3600 * 2,
digest_config=Job.CacheDigestConfig(
include_paths=[
"./src",
@ -57,85 +54,9 @@ build_jobs = Job.Config(
"./docker/packager/packager",
"./rust",
"./tests/ci/version_helper.py",
"./ci/jobs/build_clickhouse.py",
],
),
).parametrize(
parameter=["amd_debug", "amd_release", "arm_release", "arm_asan"],
provides=[
[ArtifactNames.CH_AMD_DEBUG],
[ArtifactNames.CH_AMD_RELEASE],
[ArtifactNames.CH_ARM_RELEASE],
[ArtifactNames.CH_ARM_ASAN],
],
runs_on=[
[RunnerLabels.BUILDER_AMD],
[RunnerLabels.BUILDER_AMD],
[RunnerLabels.BUILDER_ARM],
[RunnerLabels.BUILDER_ARM],
],
)
stateless_tests_jobs = Job.Config(
name=JobNames.STATELESS,
runs_on=[RunnerLabels.BUILDER_AMD],
command="python3 ./ci/jobs/functional_stateless_tests.py --test-options {PARAMETER}",
# many tests expect to see "/var/lib/clickhouse" in various output lines - add mount for now, consider creating this dir in docker file
run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined",
digest_config=Job.CacheDigestConfig(
include_paths=[
"./ci/jobs/functional_stateless_tests.py",
],
),
).parametrize(
parameter=[
"amd_debug,parallel",
"amd_debug,non-parallel",
"amd_release,parallel",
"amd_release,non-parallel",
"arm_asan,parallel",
"arm_asan,non-parallel",
],
runs_on=[
[RunnerLabels.BUILDER_AMD],
[RunnerLabels.FUNC_TESTER_AMD],
[RunnerLabels.BUILDER_AMD],
[RunnerLabels.FUNC_TESTER_AMD],
[RunnerLabels.BUILDER_ARM],
[RunnerLabels.FUNC_TESTER_ARM],
],
requires=[
[ArtifactNames.CH_AMD_DEBUG],
[ArtifactNames.CH_AMD_DEBUG],
[ArtifactNames.CH_AMD_RELEASE],
[ArtifactNames.CH_AMD_RELEASE],
[ArtifactNames.CH_ARM_ASAN],
[ArtifactNames.CH_ARM_ASAN],
],
)
stateful_tests_jobs = Job.Config(
name=JobNames.STATEFUL,
runs_on=[RunnerLabels.BUILDER_AMD],
command="python3 ./ci/jobs/functional_stateful_tests.py --test-options {PARAMETER}",
# many tests expect to see "/var/lib/clickhouse"
# some tests expect to see "/var/log/clickhouse"
run_in_docker="clickhouse/stateless-test+--security-opt seccomp=unconfined",
digest_config=Job.CacheDigestConfig(
include_paths=[
"./ci/jobs/functional_stateful_tests.py",
],
),
).parametrize(
parameter=[
"amd_debug,parallel",
],
runs_on=[
[RunnerLabels.BUILDER_AMD],
],
requires=[
[ArtifactNames.CH_AMD_DEBUG],
],
provides=[ArtifactNames.ch_debug_binary],
)
workflow = Workflow.Config(
@ -145,31 +66,14 @@ workflow = Workflow.Config(
jobs=[
style_check_job,
fast_test_job,
*build_jobs,
*stateless_tests_jobs,
*stateful_tests_jobs,
job_build_amd_debug,
],
artifacts=[
Artifact.Config(
name=ArtifactNames.CH_AMD_DEBUG,
name=ArtifactNames.ch_debug_binary,
type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
),
Artifact.Config(
name=ArtifactNames.CH_AMD_RELEASE,
type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
),
Artifact.Config(
name=ArtifactNames.CH_ARM_RELEASE,
type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
),
Artifact.Config(
name=ArtifactNames.CH_ARM_ASAN,
type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
),
)
],
dockers=DOCKERS,
secrets=SECRETS,
@ -180,14 +84,11 @@ workflow = Workflow.Config(
WORKFLOWS = [
workflow,
]
] # type: List[Workflow.Config]
# if __name__ == "__main__":
# # local job test inside praktika environment
# from praktika.runner import Runner
# from praktika.digest import Digest
#
# print(Digest().calc_job_digest(amd_debug_build_job))
#
# Runner().run(workflow, fast_test_job, docker="fasttest", local_run=True)
if __name__ == "__main__":
# local job test inside praktika environment
from praktika.runner import Runner
Runner().run(workflow, fast_test_job, docker="fasttest", dummy_env=True)

1
cmake/cpu_features.cmake
View File

@ -74,6 +74,7 @@ elseif (ARCH_AARCH64)
# introduced as optional, either in v8.2 [7] or in v8.4 [8].
# rcpc: Load-Acquire RCpc Register. Better support of release/acquire of atomics. Good for allocators and high contention code.
# Optional in v8.2, mandatory in v8.3 [9]. Supported in Graviton >=2, Azure and GCP instances.
# bf16: Bfloat16, a half-precision floating point format developed by Google Brain. Optional in v8.2, mandatory in v8.6.
#
# [1] https://github.com/aws/aws-graviton-getting-started/blob/main/c-c%2B%2B.md
# [2] https://community.arm.com/arm-community-blogs/b/tools-software-ides-blog/posts/making-the-most-of-the-arm-architecture-in-gcc-10

2
contrib/CMakeLists.txt vendored
View File

@ -219,6 +219,8 @@ add_contrib (prometheus-protobufs-cmake prometheus-protobufs prometheus-protobuf
add_contrib (numactl-cmake numactl)
add_contrib (jwt-cpp-cmake jwt-cpp)
# Put all targets defined here and in subdirectories under "contrib/<immediate-subdir>" folders in GUI-based IDEs.
# Some of third-party projects may override CMAKE_FOLDER or FOLDER property of their targets, so they would not appear
# in "contrib/..." as originally planned, so we workaround this by fixing FOLDER properties of all targets manually,

1
contrib/jwt-cpp vendored Submodule

@ -0,0 +1 @@
Subproject commit a6927cb8140858c34e05d1a954626b9849fbcdfc

23
contrib/jwt-cpp-cmake/CMakeLists.txt Normal file
View File

@ -0,0 +1,23 @@
set(ENABLE_JWT_CPP_DEFAULT OFF)
if(ENABLE_LIBRARIES AND CLICKHOUSE_CLOUD)
set(ENABLE_JWT_CPP_DEFAULT ON)
endif()
option(ENABLE_JWT_CPP "Enable jwt-cpp library" ${ENABLE_JWT_CPP_DEFAULT})
if (NOT ENABLE_JWT_CPP)
message(STATUS "Not using jwt-cpp")
return()
endif()
if(ENABLE_JWT_CPP)
if(NOT TARGET OpenSSL::Crypto)
message (${RECONFIGURE_MESSAGE_LEVEL} "Can't use jwt-cpp without OpenSSL")
endif()
endif()
set (JWT_CPP_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/jwt-cpp/include")
add_library (_jwt-cpp INTERFACE)
target_include_directories(_jwt-cpp SYSTEM BEFORE INTERFACE ${JWT_CPP_INCLUDE_DIR})
add_library(ch_contrib::jwt-cpp ALIAS _jwt-cpp)

9
docker/keeper/Dockerfile
View File

@ -31,14 +31,14 @@ COPY entrypoint.sh /entrypoint.sh
ARG TARGETARCH
RUN arch=${TARGETARCH:-amd64} \
&& case $arch in \
amd64) mkdir -p /lib64 && ln -sf /lib/ld-2.31.so /lib64/ld-linux-x86-64.so.2 ;; \
arm64) ln -sf /lib/ld-2.31.so /lib/ld-linux-aarch64.so.1 ;; \
amd64) mkdir -p /lib64 && ln -sf /lib/ld-2.35.so /lib64/ld-linux-x86-64.so.2 ;; \
arm64) ln -sf /lib/ld-2.35.so /lib/ld-linux-aarch64.so.1 ;; \
esac
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
ARG VERSION="24.10.1.2812"
ARG VERSION="24.10.2.80"
ARG PACKAGES="clickhouse-keeper"
ARG DIRECT_DOWNLOAD_URLS=""
@ -86,7 +86,8 @@ RUN arch=${TARGETARCH:-amd64} \
ARG DEFAULT_CONFIG_DIR="/etc/clickhouse-keeper"
ARG DEFAULT_DATA_DIR="/var/lib/clickhouse-keeper"
ARG DEFAULT_LOG_DIR="/var/log/clickhouse-keeper"
RUN mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" \
RUN clickhouse-keeper --version \
&& mkdir -p "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}" \
&& chown clickhouse:clickhouse "${DEFAULT_DATA_DIR}" \
&& chown root:clickhouse "${DEFAULT_LOG_DIR}" \
&& chmod ugo+Xrw -R "${DEFAULT_DATA_DIR}" "${DEFAULT_LOG_DIR}" "${DEFAULT_CONFIG_DIR}"

2
docker/server/Dockerfile.alpine
View File

@ -35,7 +35,7 @@ RUN arch=${TARGETARCH:-amd64} \
# lts / testing / prestable / etc
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="https://packages.clickhouse.com/tgz/${REPO_CHANNEL}"
ARG VERSION="24.10.1.2812"
ARG VERSION="24.10.2.80"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
ARG DIRECT_DOWNLOAD_URLS=""

2
docker/server/Dockerfile.ubuntu
View File

@ -28,7 +28,7 @@ RUN sed -i "s|http://archive.ubuntu.com|${apt_archive}|g" /etc/apt/sources.list
ARG REPO_CHANNEL="stable"
ARG REPOSITORY="deb [signed-by=/usr/share/keyrings/clickhouse-keyring.gpg] https://packages.clickhouse.com/deb ${REPO_CHANNEL} main"
ARG VERSION="24.10.1.2812"
ARG VERSION="24.10.2.80"
ARG PACKAGES="clickhouse-client clickhouse-server clickhouse-common-static"
#docker-official-library:off

61
docs/changelogs/v24.10.2.80-stable.md Normal file
View File

@ -0,0 +1,61 @@
---
sidebar_position: 1
sidebar_label: 2024
---
# 2024 Changelog
### ClickHouse release v24.10.2.80-stable (96b80057159) FIXME as compared to v24.10.1.2812-stable (9cd0a3738d5)
#### Backward Incompatible Change
* Backported in [#71363](https://github.com/ClickHouse/ClickHouse/issues/71363): Fix possible error `No such file or directory` due to unescaped special symbols in files for JSON subcolumns. [#71182](https://github.com/ClickHouse/ClickHouse/pull/71182) ([Pavel Kruglov](https://github.com/Avogar)).
#### Performance Improvement
* Backported in [#71852](https://github.com/ClickHouse/ClickHouse/issues/71852): Improve the performance and accuracy of system.query_metric_log collection interval by reducing the critical region. [#71473](https://github.com/ClickHouse/ClickHouse/pull/71473) ([Pablo Marcos](https://github.com/pamarcos)).
#### Improvement
* Backported in [#71495](https://github.com/ClickHouse/ClickHouse/issues/71495): Enable `parallel_replicas_local_plan` by default. Building a full-fledged local plan on the query initiator improves parallel replicas performance with less resource consumption, provides opportunities to apply more query optimizations. [#70171](https://github.com/ClickHouse/ClickHouse/pull/70171) ([Igor Nikonov](https://github.com/devcrafter)).
* Backported in [#71985](https://github.com/ClickHouse/ClickHouse/issues/71985): Fixes RIGHT / FULL joins in queries with parallel replicas. Now, RIGHT joins can be executed with parallel replicas (right table reading is distributed). FULL joins can't be parallelized among nodes, - executed locally. [#71162](https://github.com/ClickHouse/ClickHouse/pull/71162) ([Igor Nikonov](https://github.com/devcrafter)).
* Backported in [#71670](https://github.com/ClickHouse/ClickHouse/issues/71670): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#71940](https://github.com/ClickHouse/ClickHouse/issues/71940): Update `HostResolver` 3 times in a `history` period. [#71863](https://github.com/ClickHouse/ClickHouse/pull/71863) ([Sema Checherinda](https://github.com/CheSema)).
* Backported in [#71922](https://github.com/ClickHouse/ClickHouse/issues/71922): Allow_reorder_prewhere_conditions is on by default with old compatibility settings. [#71867](https://github.com/ClickHouse/ClickHouse/pull/71867) ([Raúl Marín](https://github.com/Algunenano)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Backported in [#71588](https://github.com/ClickHouse/ClickHouse/issues/71588): Fix mismatched aggreage function name of quantileExactWeightedInterpolated. The bug was introduced in https://github.com/ClickHouse/ClickHouse/pull/69619. cc @Algunenano. [#71168](https://github.com/ClickHouse/ClickHouse/pull/71168) ([李扬](https://github.com/taiyang-li)).
* Backported in [#71357](https://github.com/ClickHouse/ClickHouse/issues/71357): Fix bad_weak_ptr exception with Dynamic in functions comparison. [#71183](https://github.com/ClickHouse/ClickHouse/pull/71183) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#71467](https://github.com/ClickHouse/ClickHouse/issues/71467): Fix bug of memory usage increase if enable_filesystem_cache=1, but disk in storage configuration did not have any cache configuration. [#71261](https://github.com/ClickHouse/ClickHouse/pull/71261) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Backported in [#71355](https://github.com/ClickHouse/ClickHouse/issues/71355): Fix possible error "Cannot read all data" erros during deserialization of LowCardinality dictionary from Dynamic column. [#71299](https://github.com/ClickHouse/ClickHouse/pull/71299) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#71324](https://github.com/ClickHouse/ClickHouse/issues/71324): Fix incomplete cleanup of parallel output format in the client. [#71304](https://github.com/ClickHouse/ClickHouse/pull/71304) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71466](https://github.com/ClickHouse/ClickHouse/issues/71466): Added missing unescaping in named collections. Without fix clickhouse-server can't start. [#71308](https://github.com/ClickHouse/ClickHouse/pull/71308) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
* Backported in [#71393](https://github.com/ClickHouse/ClickHouse/issues/71393): Fix inconsistent AST formatting when granting wrong wildcard grants [#71309](https://github.com/ClickHouse/ClickHouse/issues/71309). [#71332](https://github.com/ClickHouse/ClickHouse/pull/71332) ([pufit](https://github.com/pufit)).
* Backported in [#71379](https://github.com/ClickHouse/ClickHouse/issues/71379): Add try/catch to data parts destructors to avoid terminate. [#71364](https://github.com/ClickHouse/ClickHouse/pull/71364) ([alesapin](https://github.com/alesapin)).
* Backported in [#71751](https://github.com/ClickHouse/ClickHouse/issues/71751): Check suspicious and experimental types in JSON type hints. [#71369](https://github.com/ClickHouse/ClickHouse/pull/71369) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#71451](https://github.com/ClickHouse/ClickHouse/issues/71451): Start memory worker thread on non-Linux OS too (fixes [#71051](https://github.com/ClickHouse/ClickHouse/issues/71051)). [#71384](https://github.com/ClickHouse/ClickHouse/pull/71384) ([Alexandre Snarskii](https://github.com/snar)).
* Backported in [#71608](https://github.com/ClickHouse/ClickHouse/issues/71608): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#71493](https://github.com/ClickHouse/ClickHouse/issues/71493): Fix crash in `mongodb` table function when passing wrong arguments (e.g. `NULL`). [#71426](https://github.com/ClickHouse/ClickHouse/pull/71426) ([Vladimir Cherkasov](https://github.com/vdimir)).
* Backported in [#71815](https://github.com/ClickHouse/ClickHouse/issues/71815): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71521](https://github.com/ClickHouse/ClickHouse/issues/71521): Fix possible error `Argument for function must be constant` (old analyzer) in case when arrayJoin can apparently appear in `WHERE` condition. Regression after https://github.com/ClickHouse/ClickHouse/pull/65414. [#71476](https://github.com/ClickHouse/ClickHouse/pull/71476) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Backported in [#71555](https://github.com/ClickHouse/ClickHouse/issues/71555): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71618](https://github.com/ClickHouse/ClickHouse/issues/71618): Analyzer fix when query inside materialized view uses IN with CTE. Closes [#65598](https://github.com/ClickHouse/ClickHouse/issues/65598). [#71538](https://github.com/ClickHouse/ClickHouse/pull/71538) ([Maksim Kita](https://github.com/kitaisreal)).
* Backported in [#71570](https://github.com/ClickHouse/ClickHouse/issues/71570): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71646](https://github.com/ClickHouse/ClickHouse/issues/71646): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
* Backported in [#71880](https://github.com/ClickHouse/ClickHouse/issues/71880): Fix LOGICAL_ERROR when doing ALTER with empty tuple. This fixes [#71647](https://github.com/ClickHouse/ClickHouse/issues/71647). [#71679](https://github.com/ClickHouse/ClickHouse/pull/71679) ([Amos Bird](https://github.com/amosbird)).
* Backported in [#71741](https://github.com/ClickHouse/ClickHouse/issues/71741): Don't transform constant set in predicates over partition columns in case of NOT IN operator. [#71695](https://github.com/ClickHouse/ClickHouse/pull/71695) ([Eduard Karacharov](https://github.com/korowa)).
* Backported in [#72012](https://github.com/ClickHouse/ClickHouse/issues/72012): Fix exception for toDayOfWeek on WHERE condition with primary key of DateTime64 type. [#71849](https://github.com/ClickHouse/ClickHouse/pull/71849) ([Yakov Olkhovskiy](https://github.com/yakov-olkhovskiy)).
* Backported in [#71897](https://github.com/ClickHouse/ClickHouse/issues/71897): Fixed filling of defaults after parsing into sparse columns. [#71854](https://github.com/ClickHouse/ClickHouse/pull/71854) ([Anton Popov](https://github.com/CurtizJ)).
* Backported in [#71955](https://github.com/ClickHouse/ClickHouse/issues/71955): Fix data race between the progress indicator and the progress table in clickhouse-client. This issue is visible when FROM INFILE is used. Intercept keystrokes during INSERT queries to toggle progress table display. [#71901](https://github.com/ClickHouse/ClickHouse/pull/71901) ([Julia Kartseva](https://github.com/jkartseva)).
* Backported in [#72006](https://github.com/ClickHouse/ClickHouse/issues/72006): Fix a crash in clickhouse-client syntax highlighting. Closes [#71864](https://github.com/ClickHouse/ClickHouse/issues/71864). [#71949](https://github.com/ClickHouse/ClickHouse/pull/71949) ([Nikolay Degterinsky](https://github.com/evillique)).
#### Build/Testing/Packaging Improvement
* Backported in [#71692](https://github.com/ClickHouse/ClickHouse/issues/71692): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Backported in [#71387](https://github.com/ClickHouse/ClickHouse/issues/71387): Remove bad test `test_system_replicated_fetches`. [#71071](https://github.com/ClickHouse/ClickHouse/pull/71071) ([Alexey Milovidov](https://github.com/alexey-milovidov)).
* Backported in [#71586](https://github.com/ClickHouse/ClickHouse/issues/71586): Fix `WITH TOTALS` in subquery with parallel replicas. [#71224](https://github.com/ClickHouse/ClickHouse/pull/71224) ([Nikita Taranov](https://github.com/nickitat)).
* Backported in [#71437](https://github.com/ClickHouse/ClickHouse/issues/71437): Ignore `No such key` exceptions in some cases. [#71236](https://github.com/ClickHouse/ClickHouse/pull/71236) ([Antonio Andelic](https://github.com/antonio2368)).
* Backported in [#71629](https://github.com/ClickHouse/ClickHouse/issues/71629): Fix compatibility with refreshable materialized views created by old clickhouse servers. [#71556](https://github.com/ClickHouse/ClickHouse/pull/71556) ([Michael Kolupaev](https://github.com/al13n321)).
* Backported in [#71805](https://github.com/ClickHouse/ClickHouse/issues/71805): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#71832](https://github.com/ClickHouse/ClickHouse/issues/71832): Closes [#71780](https://github.com/ClickHouse/ClickHouse/issues/71780). [#71818](https://github.com/ClickHouse/ClickHouse/pull/71818) ([Kseniia Sumarokova](https://github.com/kssenii)).
* Backported in [#71840](https://github.com/ClickHouse/ClickHouse/issues/71840): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).

31
docs/changelogs/v24.3.14.35-lts.md Normal file
View File

@ -0,0 +1,31 @@
---
sidebar_position: 1
sidebar_label: 2024
---
# 2024 Changelog
### ClickHouse release v24.3.14.35-lts (cfa4e62b775) FIXME as compared to v24.3.13.40-lts (7acabd77389)
#### Improvement
* Backported in [#71711](https://github.com/ClickHouse/ClickHouse/issues/71711): CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
* Backported in [#71662](https://github.com/ClickHouse/ClickHouse/issues/71662): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Backported in [#65755](https://github.com/ClickHouse/ClickHouse/issues/65755): Fix the `Expression nodes list expected 1 projection names` and `Unknown expression or identifier` errors for queries with aliases to `GLOBAL IN.`. [#64517](https://github.com/ClickHouse/ClickHouse/pull/64517) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Backported in [#71600](https://github.com/ClickHouse/ClickHouse/issues/71600): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#71842](https://github.com/ClickHouse/ClickHouse/issues/71842): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71562](https://github.com/ClickHouse/ClickHouse/issues/71562): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71731](https://github.com/ClickHouse/ClickHouse/issues/71731): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
#### Build/Testing/Packaging Improvement
* Backported in [#71697](https://github.com/ClickHouse/ClickHouse/issues/71697): Vendor in rust dependencies. [#62297](https://github.com/ClickHouse/ClickHouse/pull/62297) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71688](https://github.com/ClickHouse/ClickHouse/issues/71688): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Backported in [#71808](https://github.com/ClickHouse/ClickHouse/issues/71808): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#71834](https://github.com/ClickHouse/ClickHouse/issues/71834): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Fix bitShift test after backport. [#71861](https://github.com/ClickHouse/ClickHouse/pull/71861) ([Pablo Marcos](https://github.com/pamarcos)).
* Revert "Merge pull request [#71861](https://github.com/ClickHouse/ClickHouse/issues/71861) from pamarcos/fix-bitshift-test". [#71871](https://github.com/ClickHouse/ClickHouse/pull/71871) ([Pablo Marcos](https://github.com/pamarcos)).

37
docs/changelogs/v24.8.7.41-lts.md Normal file
View File

@ -0,0 +1,37 @@
---
sidebar_position: 1
sidebar_label: 2024
---
# 2024 Changelog
### ClickHouse release v24.8.7.41-lts (e28553d4f2b) FIXME as compared to v24.8.6.70-lts (ddb8c219771)
#### Improvement
* Backported in [#71713](https://github.com/ClickHouse/ClickHouse/issues/71713): CLICKHOUSE_PASSWORD is escaped for XML in clickhouse image's entrypoint. [#69301](https://github.com/ClickHouse/ClickHouse/pull/69301) ([aohoyd](https://github.com/aohoyd)).
* Backported in [#71666](https://github.com/ClickHouse/ClickHouse/issues/71666): When user/group is given as ID, the `clickhouse su` fails. This patch fixes it to accept `UID:GID` as well. ### Documentation entry for user-facing changes. [#71626](https://github.com/ClickHouse/ClickHouse/pull/71626) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#71936](https://github.com/ClickHouse/ClickHouse/issues/71936): Update `HostResolver` 3 times in a `history` period. [#71863](https://github.com/ClickHouse/ClickHouse/pull/71863) ([Sema Checherinda](https://github.com/CheSema)).
#### Bug Fix (user-visible misbehavior in an official stable release)
* Backported in [#71486](https://github.com/ClickHouse/ClickHouse/issues/71486): Fix `Content-Encoding` not sent in some compressed responses. [#64802](https://github.com/ClickHouse/ClickHouse/issues/64802). [#68975](https://github.com/ClickHouse/ClickHouse/pull/68975) ([Konstantin Bogdanov](https://github.com/thevar1able)).
* Backported in [#71462](https://github.com/ClickHouse/ClickHouse/issues/71462): Added missing unescaping in named collections. Without fix clickhouse-server can't start. [#71308](https://github.com/ClickHouse/ClickHouse/pull/71308) ([MikhailBurdukov](https://github.com/MikhailBurdukov)).
* Backported in [#71747](https://github.com/ClickHouse/ClickHouse/issues/71747): Check suspicious and experimental types in JSON type hints. [#71369](https://github.com/ClickHouse/ClickHouse/pull/71369) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#71604](https://github.com/ClickHouse/ClickHouse/issues/71604): Fix error Invalid number of rows in Chunk with Variant column. [#71388](https://github.com/ClickHouse/ClickHouse/pull/71388) ([Pavel Kruglov](https://github.com/Avogar)).
* Backported in [#71826](https://github.com/ClickHouse/ClickHouse/issues/71826): Fix crash with optimize_rewrite_array_exists_to_has. [#71432](https://github.com/ClickHouse/ClickHouse/pull/71432) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71517](https://github.com/ClickHouse/ClickHouse/issues/71517): Fix possible error `Argument for function must be constant` (old analyzer) in case when arrayJoin can apparently appear in `WHERE` condition. Regression after https://github.com/ClickHouse/ClickHouse/pull/65414. [#71476](https://github.com/ClickHouse/ClickHouse/pull/71476) ([Nikolai Kochetov](https://github.com/KochetovNicolai)).
* Backported in [#71551](https://github.com/ClickHouse/ClickHouse/issues/71551): Prevent crash in SortCursor with 0 columns (old analyzer). [#71494](https://github.com/ClickHouse/ClickHouse/pull/71494) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71614](https://github.com/ClickHouse/ClickHouse/issues/71614): Analyzer fix when query inside materialized view uses IN with CTE. Closes [#65598](https://github.com/ClickHouse/ClickHouse/issues/65598). [#71538](https://github.com/ClickHouse/ClickHouse/pull/71538) ([Maksim Kita](https://github.com/kitaisreal)).
* Backported in [#71566](https://github.com/ClickHouse/ClickHouse/issues/71566): Avoid crash when using a UDF in a constraint. [#71541](https://github.com/ClickHouse/ClickHouse/pull/71541) ([Raúl Marín](https://github.com/Algunenano)).
* Backported in [#71727](https://github.com/ClickHouse/ClickHouse/issues/71727): Return 0 or default char instead of throwing an error in bitShift functions in case of out of bounds. [#71580](https://github.com/ClickHouse/ClickHouse/pull/71580) ([Pablo Marcos](https://github.com/pamarcos)).
* Backported in [#71876](https://github.com/ClickHouse/ClickHouse/issues/71876): Fix LOGICAL_ERROR when doing ALTER with empty tuple. This fixes [#71647](https://github.com/ClickHouse/ClickHouse/issues/71647). [#71679](https://github.com/ClickHouse/ClickHouse/pull/71679) ([Amos Bird](https://github.com/amosbird)).
* Backported in [#71737](https://github.com/ClickHouse/ClickHouse/issues/71737): Don't transform constant set in predicates over partition columns in case of NOT IN operator. [#71695](https://github.com/ClickHouse/ClickHouse/pull/71695) ([Eduard Karacharov](https://github.com/korowa)).
* Backported in [#72002](https://github.com/ClickHouse/ClickHouse/issues/72002): Fix a crash in clickhouse-client syntax highlighting. Closes [#71864](https://github.com/ClickHouse/ClickHouse/issues/71864). [#71949](https://github.com/ClickHouse/ClickHouse/pull/71949) ([Nikolay Degterinsky](https://github.com/evillique)).
#### Build/Testing/Packaging Improvement
* Backported in [#71690](https://github.com/ClickHouse/ClickHouse/issues/71690): Improve clickhouse-server Dockerfile.ubuntu. Deprecate `CLICKHOUSE_UID/CLICKHOUSE_GID` envs. Remove `CLICKHOUSE_DOCKER_RESTART_ON_EXIT` processing to complien requirements. Consistent `clickhouse/clickhouse-server/clickhouse-keeper` execution to not have it plain in one place and `/usr/bin/clickhouse*` in another. [#71573](https://github.com/ClickHouse/ClickHouse/pull/71573) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
#### NOT FOR CHANGELOG / INSIGNIFICANT
* Backported in [#71801](https://github.com/ClickHouse/ClickHouse/issues/71801): Fix issues we face on orphane backport branches and closed release PRs, when fake-master events are sent to the check DB. [#71782](https://github.com/ClickHouse/ClickHouse/pull/71782) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).
* Backported in [#71836](https://github.com/ClickHouse/ClickHouse/issues/71836): The change has already been applied to https://github.com/docker-library/official-images/pull/17876. Backport it to every branch to have a proper `Dockerfile.ubuntu` there. [#71825](https://github.com/ClickHouse/ClickHouse/pull/71825) ([Mikhail f. Shiryaev](https://github.com/Felixoid)).

88
docs/changelogs/v24.9.3.128-stable.md Normal file
View File

File diff suppressed because one or more lines are too long

6
docs/en/engines/table-engines/integrations/s3queue.md
View File

@ -122,7 +122,7 @@ Default value: `0`.
### s3queue_polling_min_timeout_ms {#polling_min_timeout_ms}
Minimal timeout before next polling (in milliseconds).
Specifies the minimum time, in milliseconds, that ClickHouse waits before making the next polling attempt.
Possible values:
@ -132,7 +132,7 @@ Default value: `1000`.
### s3queue_polling_max_timeout_ms {#polling_max_timeout_ms}
Maximum timeout before next polling (in milliseconds).
Defines the maximum time, in milliseconds, that ClickHouse waits before initiating the next polling attempt.
Possible values:
@ -142,7 +142,7 @@ Default value: `10000`.
### s3queue_polling_backoff_ms {#polling_backoff_ms}
Polling backoff (in milliseconds).
Determines the additional wait time added to the previous polling interval when no new files are found. The next poll occurs after the sum of the previous interval and this backoff value, or the maximum interval, whichever is lower.
Possible values:

5
docs/en/engines/table-engines/mergetree-family/aggregatingmergetree.md
View File

@ -10,6 +10,11 @@ The engine inherits from [MergeTree](../../../engines/table-engines/mergetree-fa
You can use `AggregatingMergeTree` tables for incremental data aggregation, including for aggregated materialized views.
You can see an example of how to use the AggregatingMergeTree and Aggregate functions in the below video:
<div class='vimeo-container'>
<iframe width="1030" height="579" src="https://www.youtube.com/embed/pryhI4F_zqQ" title="Aggregation States in ClickHouse" frameborder="0" allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" referrerpolicy="strict-origin-when-cross-origin" allowfullscreen></iframe>
</div>
The engine processes all columns with the following types:
## [AggregateFunction](../../../sql-reference/data-types/aggregatefunction.md)

3
docs/en/engines/table-engines/mergetree-family/mergetree.md
View File

@ -684,8 +684,7 @@ If you perform the `SELECT` query between merges, you may get expired data. To a
**See Also**
- [ttl_only_drop_parts](/docs/en/operations/settings/settings.md/#ttl_only_drop_parts) setting
- [ttl_only_drop_parts](/docs/en/operations/settings/merge-tree-settings#ttl_only_drop_parts) setting
## Disk types

2
docs/en/getting-started/install.md
View File

@ -16,7 +16,7 @@ You have four options for getting up and running with ClickHouse:
- **[ClickHouse Cloud](https://clickhouse.com/cloud/):** The official ClickHouse as a service, - built by, maintained and supported by the creators of ClickHouse
- **[Quick Install](#quick-install):** an easy-to-download binary for testing and developing with ClickHouse
- **[Production Deployments](#available-installation-options):** ClickHouse can run on any Linux, FreeBSD, or macOS with x86-64, modern ARM (ARMv8.2-A up), or PowerPC64LE CPU architecture
- **[Docker Image](https://hub.docker.com/r/clickhouse/clickhouse-server/):** use the official Docker image in Docker Hub
- **[Docker Image](https://hub.docker.com/_/clickhouse):** use the official Docker image in Docker Hub
## ClickHouse Cloud

1
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -1643,6 +1643,7 @@ You can specify the log format that will be outputted in the console log. Curren
```json
{
"date_time_utc": "2024-11-06T09:06:09Z",
"date_time": "1650918987.180175",
"thread_name": "#1",
"thread_id": "254545",

20
docs/en/operations/settings/merge-tree-settings.md
View File

@ -78,6 +78,16 @@ If `min_merge_bytes_to_use_direct_io = 0`, then direct I/O is disabled.
Default value: `10 * 1024 * 1024 * 1024` bytes.
## ttl_only_drop_parts
Controls whether data parts are fully dropped in MergeTree tables when all rows in that part have expired according to their `TTL` settings.
When `ttl_only_drop_parts` is disabled (by default), only the rows that have expired based on their TTL settings are removed.
When `ttl_only_drop_parts` is enabled, the entire part is dropped if all rows in that part have expired according to their `TTL` settings.
Default value: 0.
## merge_with_ttl_timeout
Minimum delay in seconds before repeating a merge with delete TTL.
@ -1120,3 +1130,13 @@ SELECT * FROM example WHERE key = 'xxx' ORDER BY time DESC LIMIT 10;
By using `ORDER BY time DESC` in the query, `ReadInOrder` is applied.
**Default Value:** false
## cache_populated_by_fetch
A Cloud only setting.
When `cache_populated_by_fetch` is disabled (the default setting), new data parts are loaded into the cache only when a query is run that requires those parts.
If enabled, `cache_populated_by_fetch` will instead cause all nodes to load new data parts from storage into their cache without requiring a query to trigger such an action.
Default value: 0.

2
docs/en/operations/system-tables/asynchronous_metrics.md
View File

@ -211,7 +211,7 @@ Number of threads in the server of the replicas communication protocol (without
The difference in time the thread for calculation of the asynchronous metrics was scheduled to wake up and the time it was in fact, woken up. A proxy-indicator of overall system latency and responsiveness.
### LoadAverage_*N*
### LoadAverage*N*
The whole system load, averaged with exponential smoothing over 1 minute. The load represents the number of threads across all the processes (the scheduling entities of the OS kernel), that are currently running by CPU or waiting for IO, or ready to run but not being scheduled at this point of time. This number includes all the processes, not only clickhouse-server. The number can be greater than the number of CPU cores, if the system is overloaded, and many processes are ready to run but waiting for CPU or IO.

4
docs/en/sql-reference/aggregate-functions/index.md
View File

@ -75,7 +75,7 @@ FROM t_null_big
└────────────────────┴─────────────────────┘
```
Also you can use [Tuple](/docs/en/sql-reference/data-types/tuple.md) to work around NULL skipping behavior. The a `Tuple` that contains only a `NULL` value is not `NULL`, so the aggregate functions won't skip that row because of that `NULL` value.
Also you can use [Tuple](/docs/en/sql-reference/data-types/tuple.md) to work around NULL skipping behavior. A `Tuple` that contains only a `NULL` value is not `NULL`, so the aggregate functions won't skip that row because of that `NULL` value.
```sql
SELECT
@ -110,7 +110,7 @@ GROUP BY v
└──────┴─────────┴──────────┘
```
And here is an example of of first_value with `RESPECT NULLS` where we can see that NULL inputs are respected and it will return the first value read, whether it's NULL or not:
And here is an example of first_value with `RESPECT NULLS` where we can see that NULL inputs are respected and it will return the first value read, whether it's NULL or not:
```sql
SELECT

51
docs/en/sql-reference/aggregate-functions/reference/any.md
View File

@ -5,7 +5,15 @@ sidebar_position: 102
# any
Selects the first encountered value of a column, ignoring any `NULL` values.
Selects the first encountered value of a column.
:::warning
As a query can be executed in arbitrary order, the result of this function is non-deterministic.
If you need an arbitrary but deterministic result, use functions [`min`](../reference/min.md) or [`max`](../reference/max.md).
:::
By default, the function never returns NULL, i.e. ignores NULL values in the input column.
However, if the function is used with the `RESPECT NULLS` modifier, it returns the first value reads no matter if NULL or not.
**Syntax**
@ -13,46 +21,51 @@ Selects the first encountered value of a column, ignoring any `NULL` values.
any(column) [RESPECT NULLS]
```
Aliases: `any_value`, [`first_value`](../reference/first_value.md).
Aliases `any(column)` (without `RESPECT NULLS`)
- `any_value`
- [`first_value`](../reference/first_value.md).
Alias for `any(column) RESPECT NULLS`
- `anyRespectNulls`, `any_respect_nulls`
- `firstValueRespectNulls`, `first_value_respect_nulls`
- `anyValueRespectNulls`, `any_value_respect_nulls`
**Parameters**
- `column`: The column name.
**Returned value**
:::note
Supports the `RESPECT NULLS` modifier after the function name. Using this modifier will ensure the function selects the first value passed, regardless of whether it is `NULL` or not.
:::
The first value encountered.
:::note
The return type of the function is the same as the input, except for LowCardinality which is discarded. This means that given no rows as input it will return the default value of that type (0 for integers, or Null for a Nullable() column). You might use the `-OrNull` [combinator](../../../sql-reference/aggregate-functions/combinators.md) ) to modify this behaviour.
:::
:::warning
The query can be executed in any order and even in a different order each time, so the result of this function is indeterminate.
To get a determinate result, you can use the [`min`](../reference/min.md) or [`max`](../reference/max.md) function instead of `any`.
The return type of the function is the same as the input, except for LowCardinality which is discarded.
This means that given no rows as input it will return the default value of that type (0 for integers, or Null for a Nullable() column).
You might use the `-OrNull` [combinator](../../../sql-reference/aggregate-functions/combinators.md) ) to modify this behaviour.
:::
**Implementation details**
In some cases, you can rely on the order of execution. This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`.
In some cases, you can rely on the order of execution.
This applies to cases when `SELECT` comes from a subquery that uses `ORDER BY`.
When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions. In other words, each column selected from the table must be used either in keys or inside aggregate functions. To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
When a `SELECT` query has the `GROUP BY` clause or at least one aggregate function, ClickHouse (in contrast to MySQL) requires that all expressions in the `SELECT`, `HAVING`, and `ORDER BY` clauses be calculated from keys or from aggregate functions.
In other words, each column selected from the table must be used either in keys or inside aggregate functions.
To get behavior like in MySQL, you can put the other columns in the `any` aggregate function.
**Example**
Query:
```sql
CREATE TABLE any_nulls (city Nullable(String)) ENGINE=Log;
CREATE TABLE tab (city Nullable(String)) ENGINE=Memory;
INSERT INTO any_nulls (city) VALUES (NULL), ('Amsterdam'), ('New York'), ('Tokyo'), ('Valencia'), (NULL);
INSERT INTO tab (city) VALUES (NULL), ('Amsterdam'), ('New York'), ('Tokyo'), ('Valencia'), (NULL);
SELECT any(city) FROM any_nulls;
SELECT any(city), anyRespectNulls(city) FROM tab;
```
```response
┌─any(city)─┐
│ Amsterdam │
└───────────┘
┌─any(city)─┬─anyRespectNulls(city)─┐
│ Amsterdam │ ᴺᵁᴸᴸ │
└───────────┴───────────────────────┘
```

33
docs/en/sql-reference/aggregate-functions/reference/anylast.md
View File

@ -5,7 +5,15 @@ sidebar_position: 105
# anyLast
Selects the last value encountered, ignoring any `NULL` values by default. The result is just as indeterminate as for the [any](../../../sql-reference/aggregate-functions/reference/any.md) function.
Selects the last encountered value of a column.
:::warning
As a query can be executed in arbitrary order, the result of this function is non-deterministic.
If you need an arbitrary but deterministic result, use functions [`min`](../reference/min.md) or [`max`](../reference/max.md).
:::
By default, the function never returns NULL, i.e. ignores NULL values in the input column.
However, if the function is used with the `RESPECT NULLS` modifier, it returns the first value reads no matter if NULL or not.
**Syntax**
@ -13,13 +21,16 @@ Selects the last value encountered, ignoring any `NULL` values by default. The r
anyLast(column) [RESPECT NULLS]
```
Alias `anyLast(column)` (without `RESPECT NULLS`)
- [`last_value`](../reference/last_value.md).
Aliases for `anyLast(column) RESPECT NULLS`
- `anyLastRespectNulls`, `anyLast_respect_nulls`
- `lastValueRespectNulls`, `last_value_respect_nulls`
**Parameters**
- `column`: The column name.
:::note
Supports the `RESPECT NULLS` modifier after the function name. Using this modifier will ensure the function selects the last value passed, regardless of whether it is `NULL` or not.
:::
**Returned value**
- The last value encountered.
@ -29,15 +40,15 @@ Supports the `RESPECT NULLS` modifier after the function name. Using this modifi
Query:
```sql
CREATE TABLE any_last_nulls (city Nullable(String)) ENGINE=Log;
CREATE TABLE tab (city Nullable(String)) ENGINE=Memory;
INSERT INTO any_last_nulls (city) VALUES ('Amsterdam'),(NULL),('New York'),('Tokyo'),('Valencia'),(NULL);
INSERT INTO tab (city) VALUES ('Amsterdam'),(NULL),('New York'),('Tokyo'),('Valencia'),(NULL);
SELECT anyLast(city) FROM any_last_nulls;
SELECT anyLast(city), anyLastRespectNulls(city) FROM tab;
```
```response
┌─anyLast(city)─┐
│ Valencia │
└───────────────┘
┌─anyLast(city)─┬─anyLastRespectNulls(city)─┐
│ Valencia │ ᴺᵁᴸᴸ │
└───────────────┴───────────────────────────┘
```

6
docs/en/sql-reference/functions/date-time-functions.md
View File

@ -4489,9 +4489,9 @@ Using replacement fields, you can define a pattern for the resulting string.
| k | clockhour of day (1~24) | number | 24 |
| m | minute of hour | number | 30 |
| s | second of minute | number | 55 |
| S | fraction of second (not supported yet) | number | 978 |
| z | time zone (short name not supported yet) | text | Pacific Standard Time; PST |
| Z | time zone offset/id (not supported yet) | zone | -0800; -08:00; America/Los_Angeles |
| S | fraction of second | number | 978 |
| z | time zone | text | Eastern Standard Time; EST |
| Z | time zone offset | zone | -0800; -0812 |
| ' | escape for text | delimiter | |
| '' | single quote | literal | ' |

52
docs/en/sql-reference/functions/type-conversion-functions.md
View File

@ -6791,7 +6791,7 @@ parseDateTime(str[, format[, timezone]])
**Returned value(s)**
Returns DateTime values parsed from input string according to a MySQL style format string.
Return a [DateTime](../data-types/datetime.md) value parsed from the input string according to a MySQL-style format string.
**Supported format specifiers**
@ -6840,7 +6840,7 @@ parseDateTimeInJodaSyntax(str[, format[, timezone]])
**Returned value(s)**
Returns DateTime values parsed from input string according to a Joda style format.
Return a [DateTime](../data-types/datetime.md) value parsed from the input string according to a Joda-style format string.
**Supported format specifiers**
@ -6867,9 +6867,55 @@ Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that
Same as for [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax) except that it returns `NULL` when it encounters a date format that cannot be processed.
## parseDateTime64
Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datetime64.md) according to a [MySQL format string](https://dev.mysql.com/doc/refman/8.0/en/date-and-time-functions.html#function_date-format).
**Syntax**
``` sql
parseDateTime64(str[, format[, timezone]])
```
**Arguments**
- `str` — The String to be parsed.
- `format` — The format string. Optional. `%Y-%m-%d %H:%i:%s.%f` if not specified.
- `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional.
**Returned value(s)**
Return a [DateTime64](../data-types/datetime64.md) value parsed from the input string according to a MySQL-style format string.
The precision of the returned value is 6.
## parseDateTime64OrZero
Same as for [parseDateTime64](#parsedatetime64) except that it returns zero date when it encounters a date format that cannot be processed.
## parseDateTime64OrNull
Same as for [parseDateTime64](#parsedatetime64) except that it returns `NULL` when it encounters a date format that cannot be processed.
## parseDateTime64InJodaSyntax
Similar to [parseDateTimeInJodaSyntax](#parsedatetimeinjodasyntax). Differently, it returns a value of type [DateTime64](../data-types/datetime64.md).
Converts a [String](../data-types/string.md) to [DateTime64](../data-types/datetime64.md) according to a [Joda format string](https://joda-time.sourceforge.net/apidocs/org/joda/time/format/DateTimeFormat.html).
**Syntax**
``` sql
parseDateTime64InJodaSyntax(str[, format[, timezone]])
```
**Arguments**
- `str` — The String to be parsed.
- `format` — The format string. Optional. `yyyy-MM-dd HH:mm:ss` if not specified.
- `timezone` — [Timezone](/docs/en/operations/server-configuration-parameters/settings.md#timezone). Optional.
**Returned value(s)**
Return a [DateTime64](../data-types/datetime64.md) value parsed from the input string according to a Joda-style format string.
The precision of the returned value equal to the number of `S` placeholders in the format string (but at most 6).
## parseDateTime64InJodaSyntaxOrZero

46
docs/en/sql-reference/statements/check-grant.md Normal file
View File

@ -0,0 +1,46 @@
---
slug: /en/sql-reference/statements/check-grant
sidebar_position: 56
sidebar_label: CHECK GRANT
title: "CHECK GRANT Statement"
---
The `CHECK GRANT` query is used to check whether the current user/role has been granted a specific privilege.
## Syntax
The basic syntax of the query is as follows:
```sql
CHECK GRANT privilege[(column_name [,...])] [,...] ON {db.table[*]|db[*].*|*.*|table[*]|*}
```
- `privilege` — Type of privilege.
## Examples
If the user used to be granted the privilege, the response`check_grant` will be `1`. Otherwise, the response `check_grant` will be `0`.
If `table_1.col1` exists and current user is granted by privilege `SELECT`/`SELECT(con)` or role(with privilege), the response is `1`.
```sql
CHECK GRANT SELECT(col1) ON table_1;
```
```text
┌─result─┐
│ 1 │
└────────┘
```
If `table_2.col2` doesn't exists, or current user is not granted by privilege `SELECT`/`SELECT(con)` or role(with privilege), the response is `0`.
```sql
CHECK GRANT SELECT(col2) ON table_2;
```
```text
┌─result─┐
│ 0 │
└────────┘
```
## Wildcard
Specifying privileges you can use asterisk (`*`) instead of a table or a database name. Please check [WILDCARD GRANTS](../../sql-reference/statements/grant.md#wildcard-grants) for wildcard rules.

69
docs/en/sql-reference/statements/explain.md
View File

@ -161,6 +161,8 @@ Settings:
- `actions` — Prints detailed information about step actions. Default: 0.
- `json` — Prints query plan steps as a row in [JSON](../../interfaces/formats.md#json) format. Default: 0. It is recommended to use [TSVRaw](../../interfaces/formats.md#tabseparatedraw) format to avoid unnecessary escaping.
When `json=1` step names will contain an additional suffix with unique step identifier.
Example:
```sql
@ -194,30 +196,25 @@ EXPLAIN json = 1, description = 0 SELECT 1 UNION ALL SELECT 2 FORMAT TSVRaw;
{
"Plan": {
"Node Type": "Union",
"Node Id": "Union_10",
"Plans": [
{
"Node Type": "Expression",
"Node Id": "Expression_13",
"Plans": [
{
"Node Type": "SettingQuotaAndLimits",
"Plans": [
{
"Node Type": "ReadFromStorage"
}
]
"Node Type": "ReadFromStorage",
"Node Id": "ReadFromStorage_0"
}
]
},
{
"Node Type": "Expression",
"Node Id": "Expression_16",
"Plans": [
{
"Node Type": "SettingQuotaAndLimits",
"Plans": [
{
"Node Type": "ReadFromStorage"
}
]
"Node Type": "ReadFromStorage",
"Node Id": "ReadFromStorage_4"
}
]
}
@ -249,6 +246,7 @@ EXPLAIN json = 1, description = 0, header = 1 SELECT 1, 2 + dummy;
{
"Plan": {
"Node Type": "Expression",
"Node Id": "Expression_5",
"Header": [
{
"Name": "1",
@ -259,18 +257,10 @@ EXPLAIN json = 1, description = 0, header = 1 SELECT 1, 2 + dummy;
"Type": "UInt16"
}
],
"Plans": [
{
"Node Type": "SettingQuotaAndLimits",
"Header": [
{
"Name": "dummy",
"Type": "UInt8"
}
],
"Plans": [
{
"Node Type": "ReadFromStorage",
"Node Id": "ReadFromStorage_0",
"Header": [
{
"Name": "dummy",
@ -280,8 +270,6 @@ EXPLAIN json = 1, description = 0, header = 1 SELECT 1, 2 + dummy;
}
]
}
]
}
}
]
```
@ -351,17 +339,31 @@ EXPLAIN json = 1, actions = 1, description = 0 SELECT 1 FORMAT TSVRaw;
{
"Plan": {
"Node Type": "Expression",
"Node Id": "Expression_5",
"Expression": {
"Inputs": [],
"Inputs": [
{
"Name": "dummy",
"Type": "UInt8"
}
],
"Actions": [
{
"Node Type": "Column",
"Node Type": "INPUT",
"Result Type": "UInt8",
"Result Type": "Column",
"Result Name": "dummy",
"Arguments": [0],
"Removed Arguments": [0],
"Result": 0
},
{
"Node Type": "COLUMN",
"Result Type": "UInt8",
"Result Name": "1",
"Column": "Const(UInt8)",
"Arguments": [],
"Removed Arguments": [],
"Result": 0
"Result": 1
}
],
"Outputs": [
@ -370,17 +372,12 @@ EXPLAIN json = 1, actions = 1, description = 0 SELECT 1 FORMAT TSVRaw;
"Type": "UInt8"
}
],
"Positions": [0],
"Project Input": true
"Positions": [1]
},
"Plans": [
{
"Node Type": "SettingQuotaAndLimits",
"Plans": [
{
"Node Type": "ReadFromStorage"
}
]
"Node Type": "ReadFromStorage",
"Node Id": "ReadFromStorage_0"
}
]
}
@ -396,6 +393,8 @@ Settings:
- `graph` — Prints a graph described in the [DOT](https://en.wikipedia.org/wiki/DOT_(graph_description_language)) graph description language. Default: 0.
- `compact` — Prints graph in compact mode if `graph` setting is enabled. Default: 1.
When `compact=0` and `graph=1` processor names will contain an additional suffix with unique processor identifier.
Example:
```sql

81
docs/en/sql-reference/statements/select/except.md
View File

@ -5,9 +5,14 @@ sidebar_label: EXCEPT
# EXCEPT Clause
The `EXCEPT` clause returns only those rows that result from the first query without the second. The queries must match the number of columns, order, and type. The result of `EXCEPT` can contain duplicate rows.
The `EXCEPT` clause returns only those rows that result from the first query without the second.
Multiple `EXCEPT` statements are executed left to right if parenthesis are not specified. The `EXCEPT` operator has the same priority as the `UNION` clause and lower priority than the `INTERSECT` clause.
- Both queries must have the same number of columns in the same order and data type.
- The result of `EXCEPT` can contain duplicate rows. Use `EXCEPT DISTINCT` if this is not desirable.
- Multiple `EXCEPT` statements are executed from left to right if parentheses are not specified.
- The `EXCEPT` operator has the same priority as the `UNION` clause and lower priority than the `INTERSECT` clause.
## Syntax
``` sql
SELECT column1 [, column2 ]
@ -19,18 +24,33 @@ EXCEPT
SELECT column1 [, column2 ]
FROM table2
[WHERE condition]
```
The condition could be any expression based on your requirements.
Additionally, `EXCEPT()` can be used to exclude columns from a result in the same table, as is possible with BigQuery (Google Cloud), using the following syntax:
```sql
SELECT column1 [, column2 ] EXCEPT (column3 [, column4])
FROM table1
[WHERE condition]
```
## Examples
The examples in this section demonstrate usage of the `EXCEPT` clause.
### Filtering Numbers Using the `EXCEPT` Clause
Here is a simple example that returns the numbers 1 to 10 that are _not_ a part of the numbers 3 to 8:
Query:
``` sql
SELECT number FROM numbers(1,10) EXCEPT SELECT number FROM numbers(3,6);
SELECT number
FROM numbers(1, 10)
EXCEPT
SELECT number
FROM numbers(3, 6)
```
Result:
@ -44,7 +64,53 @@ Result:
└────────┘
```
`EXCEPT` and `INTERSECT` can often be used interchangeably with different Boolean logic, and they are both useful if you have two tables that share a common column (or columns). For example, suppose we have a few million rows of historical cryptocurrency data that contains trade prices and volume:
### Excluding Specific Columns Using `EXCEPT()`
`EXCEPT()` can be used to quickly exclude columns from a result. For instance if we want to select all columns from a table, except a few select columns as shown in the example below:
Query:
```sql
SHOW COLUMNS IN system.settings
SELECT * EXCEPT (default, alias_for, readonly, description)
FROM system.settings
LIMIT 5
```
Result:
```response
┌─field───────┬─type─────────────────────────────────────────────────────────────────────┬─null─┬─key─┬─default─┬─extra─┐
1. │ alias_for │ String │ NO │ │ ᴺᵁᴸᴸ │ │
2. │ changed │ UInt8 │ NO │ │ ᴺᵁᴸᴸ │ │
3. │ default │ String │ NO │ │ ᴺᵁᴸᴸ │ │
4. │ description │ String │ NO │ │ ᴺᵁᴸᴸ │ │
5. │ is_obsolete │ UInt8 │ NO │ │ ᴺᵁᴸᴸ │ │
6. │ max │ Nullable(String) │ YES │ │ ᴺᵁᴸᴸ │ │
7. │ min │ Nullable(String) │ YES │ │ ᴺᵁᴸᴸ │ │
8. │ name │ String │ NO │ │ ᴺᵁᴸᴸ │ │
9. │ readonly │ UInt8 │ NO │ │ ᴺᵁᴸᴸ │ │
10. │ tier │ Enum8('Production' = 0, 'Obsolete' = 4, 'Experimental' = 8, 'Beta' = 12) │ NO │ │ ᴺᵁᴸᴸ │ │
11. │ type │ String │ NO │ │ ᴺᵁᴸᴸ │ │
12. │ value │ String │ NO │ │ ᴺᵁᴸᴸ │ │
└─────────────┴──────────────────────────────────────────────────────────────────────────┴──────┴─────┴─────────┴───────┘
┌─name────────────────────┬─value──────┬─changed─┬─min──┬─max──┬─type────┬─is_obsolete─┬─tier───────┐
1. │ dialect │ clickhouse │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ Dialect │ 0 │ Production │
2. │ min_compress_block_size │ 65536 │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ UInt64 │ 0 │ Production │
3. │ max_compress_block_size │ 1048576 │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ UInt64 │ 0 │ Production │
4. │ max_block_size │ 65409 │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ UInt64 │ 0 │ Production │
5. │ max_insert_block_size │ 1048449 │ 0 │ ᴺᵁᴸᴸ │ ᴺᵁᴸᴸ │ UInt64 │ 0 │ Production │
└─────────────────────────┴────────────┴─────────┴──────┴──────┴─────────┴─────────────┴────────────┘
```
### Using `EXCEPT` and `INTERSECT` with Cryptocurrency Data
`EXCEPT` and `INTERSECT` can often be used interchangeably with different Boolean logic, and they are both useful if you have two tables that share a common column (or columns).
For example, suppose we have a few million rows of historical cryptocurrency data that contains trade prices and volume:
Query:
```sql
CREATE TABLE crypto_prices
@ -72,6 +138,8 @@ ORDER BY trade_date DESC
LIMIT 10;
```
Result:
```response
┌─trade_date─┬─crypto_name─┬──────volume─┬────price─┬───market_cap─┬──change_1_day─┐
│ 2020-11-02 │ Bitcoin │ 30771456000 │ 13550.49 │ 251119860000 │ -0.013585099 │
@ -127,7 +195,7 @@ Result:
This means of the four cryptocurrencies we own, only Bitcoin has never dropped below $10 (based on the limited data we have here in this example).
## EXCEPT DISTINCT
### Using `EXCEPT DISTINCT`
Notice in the previous query we had multiple Bitcoin holdings in the result. You can add `DISTINCT` to `EXCEPT` to eliminate duplicate rows from the result:
@ -146,7 +214,6 @@ Result:
└─────────────┘
```
**See Also**
- [UNION](union.md#union-clause)

6
docs/en/sql-reference/window-functions/first_value.md
View File

@ -15,7 +15,7 @@ first_value (column_name) [[RESPECT NULLS] | [IGNORE NULLS]]
OVER ([[PARTITION BY grouping_column] [ORDER BY sorting_column]
[ROWS or RANGE expression_to_bound_rows_withing_the_group]] | [window_name])
FROM table_name
WINDOW window_name as ([[PARTITION BY grouping_column] [ORDER BY sorting_column])
WINDOW window_name as ([PARTITION BY grouping_column] [ORDER BY sorting_column])
```
Alias: `any`.
@ -23,6 +23,8 @@ Alias: `any`.
:::note
Using the optional modifier `RESPECT NULLS` after `first_value(column_name)` will ensure that `NULL` arguments are not skipped.
See [NULL processing](../aggregate-functions/index.md/#null-processing) for more information.
Alias: `firstValueRespectNulls`
:::
For more detail on window function syntax see: [Window Functions - Syntax](./index.md/#syntax).
@ -48,7 +50,7 @@ CREATE TABLE salaries
)
Engine = Memory;
INSERT INTO salaries FORMAT Values
INSERT INTO salaries FORMAT VALUES
('Port Elizabeth Barbarians', 'Gary Chen', 196000, 'F'),
('New Coreystad Archdukes', 'Charles Juarez', 190000, 'F'),
('Port Elizabeth Barbarians', 'Michael Stanley', 100000, 'D'),

6
docs/en/sql-reference/window-functions/last_value.md
View File

@ -23,6 +23,8 @@ Alias: `anyLast`.
:::note
Using the optional modifier `RESPECT NULLS` after `first_value(column_name)` will ensure that `NULL` arguments are not skipped.
See [NULL processing](../aggregate-functions/index.md/#null-processing) for more information.
Alias: `lastValueRespectNulls`
:::
For more detail on window function syntax see: [Window Functions - Syntax](./index.md/#syntax).
@ -33,7 +35,7 @@ For more detail on window function syntax see: [Window Functions - Syntax](./ind
**Example**
In this example the `last_value` function is used to find the highest paid footballer from a fictional dataset of salaries of Premier League football players.
In this example the `last_value` function is used to find the lowest paid footballer from a fictional dataset of salaries of Premier League football players.
Query:
@ -48,7 +50,7 @@ CREATE TABLE salaries
)
Engine = Memory;
INSERT INTO salaries FORMAT Values
INSERT INTO salaries FORMAT VALUES
('Port Elizabeth Barbarians', 'Gary Chen', 196000, 'F'),
('New Coreystad Archdukes', 'Charles Juarez', 190000, 'F'),
('Port Elizabeth Barbarians', 'Michael Stanley', 100000, 'D'),

8
docs/ja/_placeholders/api/_invitations-api-reference.md Normal file
View File

@ -0,0 +1,8 @@
---
sidebar_label: 招待
title: 招待
---
## すべての招待を一覧表示
このファイルは、ビルドプロセス中に `clickhouseapi.js` によって生成されます。内容を変更する必要がある場合は、`clickhouseapi.js` を編集してください。

9
docs/ja/_placeholders/api/_keys-api-reference.md Normal file
View File

@ -0,0 +1,9 @@
---
sidebar_label: キー
title: キー
---
## すべてのキーのリストを取得する
このファイルは、ビルドプロセス中に `clickhouseapi.js` によって生成されます。
内容を変更する必要がある場合は、`clickhouseapi.js` を編集してください。

8
docs/ja/_placeholders/api/_members-api-reference.md Normal file
View File

@ -0,0 +1,8 @@
---
sidebar_label: メンバー
title: メンバー
---
## 組織メンバーの一覧
このファイルはビルドプロセス中に`clickhouseapi.js`によって生成されます。内容を変更する必要がある場合は、`clickhouseapi.js`を編集してください。

8
docs/ja/_placeholders/api/_organizations-api-reference.md Normal file
View File

@ -0,0 +1,8 @@
---
sidebar_label: 組織
title: 組織
---
## 組織の詳細を取得する
このファイルはビルドプロセス中に `clickhouseapi.js` によって生成されます。内容を変更する必要がある場合は、`clickhouseapi.js` を編集してください。

8
docs/ja/_placeholders/api/_services-api-reference.md Normal file
View File

@ -0,0 +1,8 @@
---
sidebar_label: サービス
title: サービス
---
## 組織サービスの一覧
このファイルは、ビルドプロセス中に `clickhouseapi.js` によって生成されます。内容を変更する必要がある場合は、`clickhouseapi.js` を編集してください。

8
docs/ja/_placeholders/changelog/_index.md Normal file
View File

@ -0,0 +1,8 @@
---
slug: /ja/whats-new/changelog/
sidebar_position: 2
sidebar_label: 2024
title: 2024 Changelog
note: このファイルは `yarn new-build` によって自動生成されます。
---

41
docs/ja/_snippets/_GCS_authentication_and_bucket.md Normal file
View File

@ -0,0 +1,41 @@
<details><summary>GCS バケットと HMAC キーを作成する</summary>
### ch_bucket_us_east1
![バケットを追加](@site/docs/ja/integrations/data-ingestion/s3/images/GCS-bucket-1.png)
### ch_bucket_us_east4
![バケットを追加](@site/docs/ja/integrations/data-ingestion/s3/images/GCS-bucket-2.png)
### アクセスキーを生成する
### サービスアカウントの HMAC キーとシークレットを作成する
**Cloud Storage > Settings > Interoperability** を開き、既存の **Access key** を選択するか、**CREATE A KEY FOR A SERVICE ACCOUNT** を選択します。このガイドでは、新しいサービスアカウントの新しいキーを作成する手順を説明します。
![バケットを追加](@site/docs/ja/integrations/data-ingestion/s3/images/GCS-create-a-service-account-key.png)
### 新しいサービスアカウントを追加する
すでにサービスアカウントが存在しないプロジェクトの場合は、**CREATE NEW ACCOUNT** をクリックします。
![バケットを追加](@site/docs/ja/integrations/data-ingestion/s3/images/GCS-create-service-account-0.png)
サービスアカウントを作成するには3つのステップがあります。最初のステップでは、アカウントに意味のある名前、ID、説明を付けます。
![バケットを追加](@site/docs/ja/integrations/data-ingestion/s3/images/GCS-create-service-account-a.png)
Interoperability 設定ダイアログでは、IAM ロールとして **Storage Object Admin** ロールが推奨されます。ステップ2でそのロールを選択します。
![バケットを追加](@site/docs/ja/integrations/data-ingestion/s3/images/GCS-create-service-account-2.png)
ステップ3はオプションであり、このガイドでは使用しません。ポリシーに基づいて、ユーザーにこれらの特権を与えることができます。
![バケットを追加](@site/docs/ja/integrations/data-ingestion/s3/images/GCS-create-service-account-3.png)
サービスアカウントの HMAC キーが表示されます。この情報を保存してください。ClickHouse の設定で使用します。
![バケットを追加](@site/docs/ja/integrations/data-ingestion/s3/images/GCS-guide-key.png)
</details>

132
docs/ja/_snippets/_S3_authentication_and_bucket.md Normal file
View File

@ -0,0 +1,132 @@
<details><summary>S3バケットとIAMユーザーの作成</summary>
この記事では、AWS IAMユーザーを設定し、S3バケットを作成し、ClickHouseをそのバケットをS3ディスクとして使用するように設定する基本を説明しています。使用する権限を決定するためにセキュリティチームと協力し、これらを出発点として考えてください。
### AWS IAMユーザーの作成
この手順では、ログインユーザーではなくサービスアカウントユーザーを作成します。
1. AWS IAM 管理コンソールにログインします。
2. 「ユーザー」で、**ユーザーを追加** を選択します。
![create_iam_user_0](@site/docs/ja/_snippets/images/s3/s3-1.png)
3. ユーザー名を入力し、資格情報の種類を **アクセスキー - プログラムによるアクセス** に設定し、**次: 権限** を選択します。
![create_iam_user_1](@site/docs/ja/_snippets/images/s3/s3-2.png)
4. ユーザーをグループに追加せず、**次: タグ** を選択します。
![create_iam_user_2](@site/docs/ja/_snippets/images/s3/s3-3.png)
5. タグを追加する必要がなければ、**次: 確認** を選択します。
![create_iam_user_3](@site/docs/ja/_snippets/images/s3/s3-4.png)
6. **ユーザーを作成** を選択します。
:::note
ユーザーに権限がないという警告メッセージは無視できます。次のセクションでバケットに対してユーザーに権限が付与されます。
:::
![create_iam_user_4](@site/docs/ja/_snippets/images/s3/s3-5.png)
7. ユーザーが作成されました。**表示** をクリックし、アクセスキーとシークレットキーをコピーします。
:::note
これがシークレットアクセスキーが利用可能な唯一のタイミングですので、キーを別の場所に保存してください。
:::
![create_iam_user_5](@site/docs/ja/_snippets/images/s3/s3-6.png)
8. 閉じるをクリックし、ユーザー画面でそのユーザーを見つけます。
![create_iam_user_6](@site/docs/ja/_snippets/images/s3/s3-7.png)
9. ARN（Amazon Resource Name）をコピーし、バケットのアクセスポリシーを設定する際に使用するために保存します。
![create_iam_user_7](@site/docs/ja/_snippets/images/s3/s3-8.png)
### S3バケットの作成
1. S3バケットセクションで、**バケットの作成** を選択します。
![create_s3_bucket_0](@site/docs/ja/_snippets/images/s3/s3-9.png)
2. バケット名を入力し、他のオプションはデフォルトのままにします。
:::note
バケット名はAWS全体で一意である必要があります。組織内だけでなく、一意でない場合はエラーが発生します。
:::
3. `すべてのパブリックアクセスをブロック` を有効のままにします。パブリックアクセスは必要ありません。
![create_s3_bucket_2](@site/docs/ja/_snippets/images/s3/s3-a.png)
4. ページの下部にある **バケットの作成** を選択します。
![create_s3_bucket_3](@site/docs/ja/_snippets/images/s3/s3-b.png)
5. リンクを選択し、ARNをコピーして、バケットのアクセスポリシーを設定するときに使用するために保存します。
6. バケットが作成されたら、S3バケットリストで新しいS3バケットを見つけ、リンクを選択します。
![create_s3_bucket_4](@site/docs/ja/_snippets/images/s3/s3-c.png)
7. **フォルダを作成** を選択します。
![create_s3_bucket_5](@site/docs/ja/_snippets/images/s3/s3-d.png)
8. ClickHouse S3ディスクのターゲットとなるフォルダ名を入力し、**フォルダを作成** を選択します。
![create_s3_bucket_6](@site/docs/ja/_snippets/images/s3/s3-e.png)
9. フォルダがバケットリストに表示されるはずです。
![create_s3_bucket_7](@site/docs/ja/_snippets/images/s3/s3-f.png)
10. 新しいフォルダのチェックボックスを選択し、**URLをコピー** をクリックします。コピーしたURLは、次のセクションでのClickHouseストレージ設定で使用します。
![create_s3_bucket_8](@site/docs/ja/_snippets/images/s3/s3-g.png)
11. **権限** タブを選択し、**バケットポリシー** セクションの **編集** ボタンをクリックします。
![create_s3_bucket_9](@site/docs/ja/_snippets/images/s3/s3-h.png)
12. 以下の例のようにバケットポリシーを追加します:
```json
{
"Version": "2012-10-17",
"Id": "Policy123456",
"Statement": [
{
"Sid": "abc123",
"Effect": "Allow",
"Principal": {
"AWS": "arn:aws:iam::921234567898:user/mars-s3-user"
},
"Action": "s3:*",
"Resource": [
"arn:aws:s3:::mars-doc-test",
"arn:aws:s3:::mars-doc-test/*"
]
}
]
}
```
```response
|パラメータ | 説明 | 例 |
|----------|-------------|----------------|
|Version | ポリシーインタープリタのバージョン、そのままにしておく | 2012-10-17 |
|Sid | ユーザー定義のポリシーID | abc123 |
|Effect | ユーザー要求が許可されるか拒否されるか | Allow |
|Principal | 許可されるアカウントまたはユーザー | arn:aws:iam::921234567898:user/mars-s3-user |
|Action | バケット上で許可される操作| s3:*|
|Resource | バケット内で操作が許可されるリソース | "arn:aws:s3:::mars-doc-test", "arn:aws:s3:::mars-doc-test/*" |
```
:::note
使用する権限を決定するためにセキュリティチームと協力し、これらを出発点として考えてください。
ポリシーと設定の詳細については、AWSドキュメントをご参照ください：
https://docs.aws.amazon.com/AmazonS3/latest/userguide/access-policy-language-overview.html
:::
13. ポリシー設定を保存します。
</details>

11
docs/ja/_snippets/_add_remote_ip_access_list_detail.md Normal file
View File

@ -0,0 +1,11 @@
<details><summary>IPアクセスリストを管理する</summary>
ClickHouse Cloudのサービスリストから作業するサービスを選択し、**セキュリティ**に切り替えます。IPアクセスリストに、ClickHouse Cloudサービスに接続する必要があるリモートシステムのIPアドレスや範囲が含まれていない場合は、**エントリを追加**して問題を解決できます。
![サービスがトラフィックを許可しているか確認](@site/docs/ja/_snippets/images/ip-allow-list-check-list.png)
ClickHouse Cloudサービスに接続する必要がある個別のIPアドレス、またはアドレスの範囲を追加します。フォームを適宜修正し、**エントリを追加**し、**エントリを送信**します。
![現在のIPアドレスを追加](@site/docs/ja/_snippets/images/ip-allow-list-add-current-ip.png)
</details>

45
docs/ja/_snippets/_add_superset_detail.md Normal file
View File

@ -0,0 +1,45 @@
<details><summary>DockerでApache Supersetを起動</summary>
Supersetは、[Docker Composeを使用してローカルにSupersetをインストールする](https://superset.apache.org/docs/installation/installing-superset-using-docker-compose/)手順を提供しています。GitHubからApache Supersetリポジトリをチェックアウトした後、最新の開発コードや特定のタグを実行することができます。`pre-release`としてマークされていない最新のリリースである2.0.0をお勧めします。
`docker compose`を実行する前にいくつかのタスクを行う必要があります：
1. 公式のClickHouse Connectドライバーを追加
2. MapBox APIキーを取得し、それを環境変数として追加（任意）
3. 実行するSupersetのバージョンを指定
:::tip
以下のコマンドはGitHubリポジトリのトップレベル、`superset`から実行してください。
:::
## 公式ClickHouse Connectドライバー
SupersetデプロイメントでClickHouse Connectドライバーを利用可能にするために、ローカルのrequirementsファイルに追加します：
```bash
echo "clickhouse-connect" >> ./docker/requirements-local.txt
```
## MapBox
これは任意です。MapBox APIキーなしでSupersetで位置データをプロットできますが、キーを追加するべきというメッセージが表示され、地図の背景画像が欠けます（データポイントのみが表示され、地図の背景は表示されません）。MapBoxは無料のティアを提供していますので、利用したい場合はぜひご利用ください。
ガイドが作成するサンプルの可視化の一部は、例えば経度や緯度データなどの位置情報を使用します。SupersetはMapBoxマップのサポートを含んでいます。MapBoxの可視化を使用するには、MapBox APIキーが必要です。[MapBoxの無料ティア](https://account.mapbox.com/auth/signup/)にサインアップし、APIキーを生成してください。
APIキーをSupersetで利用可能にします：
```bash
echo "MAPBOX_API_KEY=pk.SAMPLE-Use-your-key-instead" >> docker/.env-non-dev
```
## Supersetバージョン2.0.0をデプロイ
リリース2.0.0をデプロイするには、以下を実行します：
```bash
git checkout 2.0.0
TAG=2.0.0 docker-compose -f docker-compose-non-dev.yml pull
TAG=2.0.0 docker-compose -f docker-compose-non-dev.yml up
```
</details>

11
docs/ja/_snippets/_aws_regions.md Normal file
View File

@ -0,0 +1,11 @@
| 地域 | VPC サービス名 | アベイラビリティーゾーン ID |
|------------------|--------------------------------------------------------------------|------------------------------|
|ap-south-1 | com.amazonaws.vpce.ap-south-1.vpce-svc-0a786406c7ddc3a1b | aps1-az1 aps1-az2 aps1-az3 |
|ap-southeast-1 | com.amazonaws.vpce.ap-southeast-1.vpce-svc-0a8b096ec9d2acb01 | apse1-az1 apse1-az2 apse1-az3|
|ap-southeast-2 | com.amazonaws.vpce.ap-southeast-2.vpce-svc-0ca446409b23f0c01 | apse2-az1 apse2-az2 apse2-az3|
|eu-central-1 | com.amazonaws.vpce.eu-central-1.vpce-svc-0536fc4b80a82b8ed | euc1-az2 euc1-az3 euc1-az1 |
|eu-west-1 | com.amazonaws.vpce.eu-west-1.vpce-svc-066b03c9b5f61c6fc | euw1-az2 euw1-az3 euw1-az1 |
|us-east-1 c0 | com.amazonaws.vpce.us-east-1.vpce-svc-0a0218fa75c646d81 | use1-az6 use1-az1 use1-az2 |
|us-east-1 c1 | com.amazonaws.vpce.us-east-1.vpce-svc-096c118db1ff20ea4 | use1-az6 use1-az4 use1-az2 |
|us-east-2 | com.amazonaws.vpce.us-east-2.vpce-svc-0b99748bf269a86b4 | use2-az1 use2-az2 use2-az3 |
|us-west-2 | com.amazonaws.vpce.us-west-2.vpce-svc-049bbd33f61271781 | usw2-az2 usw2-az1 usw2-az3 |

15
docs/ja/_snippets/_check_ip_access_list_detail.md Normal file
View File

@ -0,0 +1,15 @@
<details><summary>IPアクセスリストを管理する</summary>
ClickHouse Cloudのサービスリストから作業するサービスを選び、**設定**に切り替えます。
![サービスの設定](@site/docs/ja/_snippets/images/cloud-service-settings.png)
IPアクセスリストが**現在、このサービスにアクセスできるトラフィックはありません**と表示される場合は、**エントリを追加**して問題を解決できます。
![サービスがトラフィックを許可しているか確認する](@site/docs/ja/_snippets/images/ip-allow-list-check-list.png)
クイックスタートのために、ローカルのセキュリティポリシーが許可する場合は、現在のIPアドレスのみを追加することができます。これを行うには、**現在のIPを追加**を使用し、現在のIPと説明「ホームIP」でフォームを自動入力します。必要に応じてフォームを修正し、**エントリを追加**し**エントリを送信**します。
![現在のIPアドレスを追加する](@site/docs/ja/_snippets/images/ip-allow-list-add-current-ip.png)
</details>

61
docs/ja/_snippets/_clickhouse_mysql_cloud_setup.mdx Normal file
View File

@ -0,0 +1,61 @@
1. ClickHouse Cloud Serviceを作成した後、認証情報画面でMySQLタブを選択します。
![Credentials screen - Prompt](./images/mysql1.png)
2. この特定のサービスに対してMySQLインターフェースを有効にするためにスイッチを切り替えます。これにより、そのサービスでポート`3306`が公開され、ユニークなMySQLユーザー名を含むMySQL接続画面が表示されます。パスワードはサービスのデフォルトユーザーのパスワードと同じになります。
![Credentials screen - Enabled MySQL](./images/mysql2.png)
代わりに、既存のサービスに対してMySQLインターフェースを有効にするには:
3. サービスが`Running`状態であることを確認し、MySQLインターフェースを有効にするサービスの「接続文字列を表示」ボタンをクリックします。
![Connection screen - Prompt MySQL](./images/mysql3.png)
4. この特定のサービスに対してMySQLインターフェースを有効にするためにスイッチを切り替えます。これにより、デフォルトのパスワードを入力するよう求められます。
![Connection screen - Prompt MySQL](./images/mysql4.png)
5. パスワードを入力すると、このサービスのMySQL接続文字列が表示されます。
![Connection screen - MySQL Enabled](./images/mysql5.png)
## ClickHouse Cloudで複数のMySQLユーザーを作成する
デフォルトでは、`mysql4<subdomain>`という組み込みユーザーがあり、これは`default`ユーザーと同じパスワードを使用します。`<subdomain>`部分はあなたのClickHouse Cloudホスト名の最初のセグメントです。このフォーマットは、安全な接続を実装しているが[TLSハンドシェイクでSNI情報を提供しない](https://www.cloudflare.com/learning/ssl/what-is-sni)ツール（MySQLコンソールクライアントがその一例）で作業するために必要です。この場合、ユーザー名に追加のヒントを含めずには内部ルーティングを行うことができません。
これにより、MySQLインターフェースで使用する新しいユーザーを作成する際には、`mysql4<subdomain>_<username>`のフォーマットを使用することを_強くお勧めします_。ここで、`<subdomain>`はあなたのCloudサービスを識別するためのヒントであり、`<username>`は選択した任意のサフィックスです。
:::tip
ClickHouse Cloudホスト名が`foobar.us-east1.aws.clickhouse.cloud`の場合、`<subdomain>`部分は`foobar`に相当し、カスタムMySQLユーザー名は`mysql4foobar_team1`のようになります。
:::
MySQLインターフェースを使用するために追加のユーザーを作成することができます。例えば、追加の設定を適用する必要がある場合などです。
1. オプション - カスタムユーザーに適用する[設定プロフィール](https://clickhouse.com/docs/ja/sql-reference/statements/create/settings-profile)を作成します。たとえば、後で作成するユーザーで接続するときにデフォルトで適用される追加設定を持つ`my_custom_profile`:
```sql
CREATE SETTINGS PROFILE my_custom_profile SETTINGS prefer_column_name_to_alias=1;
```
`prefer_column_name_to_alias`は単なる例として使用されます。ここに他の設定を使用することもできます。
2. 以下のフォーマットを使用して[ユーザーを作成](https://clickhouse.com/docs/ja/sql-reference/statements/create/user)します: `mysql4<subdomain>_<username>` ([上記参照](#creating-multiple-mysql-users-in-clickhouse-cloud))。パスワードはダブルSHA1形式である必要があります。例:
```sql
CREATE USER mysql4foobar_team1 IDENTIFIED WITH double_sha1_password BY 'YourPassword42$';
```
または、このユーザーにカスタムプロフィールを使用したい場合:
```sql
CREATE USER mysql4foobar_team1 IDENTIFIED WITH double_sha1_password BY 'YourPassword42$' SETTINGS PROFILE 'my_custom_profile';
```
ここで、`my_custom_profile`は前に作成したプロフィールの名前です。
3. 新しいユーザーに必要なアクセス権を付与して、目的のテーブルまたはデータベースと対話できるようにします。[権限を付与](https://clickhouse.com/docs/ja/sql-reference/statements/grant)する例として、たとえば`system.query_log`のみのアクセスを付与したい場合:
```sql
GRANT SELECT ON system.query_log TO mysql4foobar_team1;
```
4. 作成したユーザーを使用して、MySQLインターフェースでClickHouse Cloudサービスに接続します。
### ClickHouse Cloudでの複数のMySQLユーザーのトラブルシューティング
新しいMySQLユーザーを作成し、MySQL CLIクライアントで接続しているときに以下のエラーが表示された場合：
```
ERROR 2013 (HY000): Lost connection to MySQL server at 'reading authorization packet', system error: 54
```
この場合、ユーザー名が`mysql4<subdomain>_<username>`形式に従っていることを確認してください。[上記](#creating-multiple-mysql-users-in-clickhouse-cloud)で説明されています。

87
docs/ja/_snippets/_clickhouse_mysql_on_premise_setup.mdx Normal file
View File

@ -0,0 +1,87 @@
ClickHouseサーバーにMySQLインターフェースを有効にする方法については[公式ドキュメント](https://clickhouse.com/docs/ja/interfaces/mysql)を参照してください。
サーバーの `config.xml` にエントリを追加することに加えて、
```xml
<clickhouse>
<mysql_port>9004</mysql_port>
</clickhouse>
```
MySQLインターフェースを利用するユーザーには、[二重SHA1パスワード暗号化](https://clickhouse.com/docs/ja/operations/settings/settings-users#user-namepassword)を使用することが**必要**です。
シェルから二重SHA1で暗号化されたランダムパスワードを生成するには以下を実行してください：
```shell
PASSWORD=$(base64 < /dev/urandom | head -c16); echo "$PASSWORD"; echo -n "$PASSWORD" | sha1sum | tr -d '-' | xxd -r -p | sha1sum | tr -d '-'
```
出力は以下のようになります：
```
LZOQYnqQN4L/T6L0
fbc958cc745a82188a51f30de69eebfc67c40ee4
```
最初の行は生成されたパスワードで、2行目はClickHouseの設定に使用するハッシュです。
以下は生成されたハッシュを使用する`mysql_user`の設定例です：
`/etc/clickhouse-server/users.d/mysql_user.xml`
```xml
<users>
<mysql_user>
<password_double_sha1_hex>fbc958cc745a82188a51f30de69eebfc67c40ee4</password_double_sha1_hex>
<networks>
<ip>::/0</ip>
</networks>
<profile>default</profile>
<quota>default</quota>
</mysql_user>
</users>
```
`password_double_sha1_hex` エントリを自分で生成した二重SHA1ハッシュに置き換えてください。
さらに、BIツールがMySQLコネクタを使用する際にデータベーススキーマを適切に調査できるように、`SHOW [FULL] COLUMNS` クエリの結果でMySQLネイティブタイプを表示するために、`use_mysql_types_in_show_columns`を使用することを推奨します。
例えば：
`/etc/clickhouse-server/users.d/mysql_user.xml`
```xml
<profiles>
<default>
<use_mysql_types_in_show_columns>1</use_mysql_types_in_show_columns>
</default>
</profiles>
```
または、デフォルト以外の異なるプロファイルに割り当てることもできます。
`mysql` バイナリが利用可能であれば、コマンドラインから接続をテストできます。以下は、サンプルのユーザー名 (`mysql_user`) とパスワード (`LZOQYnqQN4L/T6L0`) を使用したコマンドです：
```bash
mysql --protocol tcp -h localhost -u mysql_user -P 9004 --password=LZOQYnqQN4L/T6L0
```
```
mysql> show databases;
+--------------------+
| name |
+--------------------+
| INFORMATION_SCHEMA |
| default |
| information_schema |
| system |
+--------------------+
4行取得しました (0.00 sec)
4行読み込み、603.00 B、0.00156秒で、2564行/秒、377.48 KiB/秒
```
最後に、ClickHouseサーバーを希望するIPアドレスでリッスンするように設定します。例えば、`config.xml` の中で、すべてのアドレスでリッスンするために以下をアンコメントしてください：
```bash
<listen_host>::</listen_host>
```

19
docs/ja/_snippets/_cloud_backup.md Normal file
View File

@ -0,0 +1,19 @@
## クラウドのバックアップとリストア
各サービスは毎日バックアップされています。サービスの**バックアップ**タブで、サービスのバックアップリストを見ることができます。そこからバックアップをリストアしたり、バックアップを削除することができます。
![バックアップのリスト](@site/docs/ja/_snippets/images/cloud-backup-list.png)
**バックアップをリストア**アイコンをクリックすると、新しく作成されるサービスの**サービス名**を指定して、**このバックアップをリストア**できます。
![バックアップのリスト](@site/docs/ja/_snippets/images/cloud-backup-restore.png)
新しいサービスは、準備が整うまでサービスリストに**プロビジョニング**として表示されます。
![バックアップのリスト](@site/docs/ja/_snippets/images/cloud-backup-new-service.png)
新しいサービスのプロビジョニングが完了すると、接続できます。その後…
:::note
ClickHouse Cloud サービスを利用する際に、SQL クライアントで `BACKUP` および `RESTORE` コマンドを使用しないでください。クラウドのバックアップは UI から管理する必要があります。
:::

7
docs/ja/_snippets/_config-files.md Normal file
View File

@ -0,0 +1,7 @@
:::important best practices
ClickHouse Server を設定する際、設定ファイルを追加または編集するときは次のようにしてください:
- ファイルを `/etc/clickhouse-server/config.d/` ディレクトリに追加する
- ファイルを `/etc/clickhouse-server/users.d/` ディレクトリに追加する
- `/etc/clickhouse-server/config.xml` ファイルはそのままにしておく
- `/etc/clickhouse-server/users.xml` ファイルはそのままにしておく
:::

17
docs/ja/_snippets/_gather_your_details_http.mdx Normal file
View File

@ -0,0 +1,17 @@
HTTP(S) を使用して ClickHouse に接続するには、以下の情報が必要です：
- **HOST と PORT**: 通常、TLS を使用する場合のポートは 8443、TLS を使用しない場合は 8123 です。
- **データベース名**: デフォルトで `default` という名前のデータベースがありますが、接続したいデータベースの名前を使用してください。
- **ユーザー名とパスワード**: デフォルトでユーザー名は `default` です。使用ケースに適したユーザー名を使用してください。
ClickHouse Cloud サービスの詳細は、ClickHouse Cloud コンソールで確認できます。 接続するサービスを選択し、**接続** をクリックします:
![ClickHouse Cloud service connect button](@site/docs/ja/_snippets/images/cloud-connect-button.png)
**HTTPS** を選択すると、サンプルの `curl` コマンドで詳細が確認できます。
![ClickHouse Cloud HTTPS connection details](@site/docs/ja/_snippets/images/connection-details-https.png)
セルフマネージドの ClickHouse を使用している場合、接続の詳細は ClickHouse 管理者によって設定されます。

17
docs/ja/_snippets/_gather_your_details_native.md Normal file
View File

@ -0,0 +1,17 @@
ClickHouseにネイティブTCPで接続するには、次の情報が必要です。
- **HOSTとPORT**: 通常、TLSを使用している場合はポートは9440、TLSを使用していない場合は9000です。
- **データベース名**: デフォルトでは、`default`という名前のデータベースがあります。接続したいデータベースの名前を使用してください。
- **ユーザー名とパスワード**: デフォルトのユーザー名は`default`です。使用するケースに適したユーザー名を利用してください。
ClickHouse Cloudサービスの詳細は、ClickHouse Cloudコンソールで確認できます。接続するサービスを選択し、**Connect**をクリックします。
![ClickHouse Cloud service connect button](@site/docs/ja/_snippets/images/cloud-connect-button.png)
**Native** を選択すると、例として `clickhouse-client` コマンドで使用可能な詳細が表示されます。
![ClickHouse Cloud Native TCP connection details](@site/docs/ja/_snippets/images/connection-details-native.png)
セルフマネージドのClickHouseを使用している場合、接続の詳細はClickHouse管理者によって設定されます。

6
docs/ja/_snippets/_gcp_regions.md Normal file
View File

@ -0,0 +1,6 @@
| リージョン | サービスアタッチメント | プライベートDNSドメイン |
|--------------|-------------------------------------------------------------|------------------------------|
|asia-southeast1| projects/dataplane-production/regions/asia-southeast1/serviceAttachments/production-asia-southeast1-clickhouse-cloud| asia-southeast1.p.gcp.clickhouse.cloud|
|europe-west4| projects/dataplane-production/regions/europe-west4/serviceAttachments/production-europe-west4-clickhouse-cloud| europe-west4.p.gcp.clickhouse.cloud|
|us-central1| projects/dataplane-production/regions/us-central1/serviceAttachments/production-us-central1-clickhouse-cloud| us-central1.p.gcp.clickhouse.cloud|
|us-east1| projects/dataplane-production/regions/us-east1/serviceAttachments/production-us-east1-clickhouse-cloud| us-east1.p.gcp.clickhouse.cloud|

5
docs/ja/_snippets/_keeper-config-files.md Normal file
View File

@ -0,0 +1,5 @@
:::important ベストプラクティス
ClickHouse Keeperを構成するために設定ファイルを編集する際には、以下を行うべきです:
- `/etc/clickhouse-keeper/keeper_config.xml` をバックアップする
- `/etc/clickhouse-keeper/keeper_config.xml` ファイルを編集する
:::

Some files were not shown because too many files have changed in this diff Show More