thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-24 16:42:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #72231 from ClickHouse/revert-71995-stress_with_praktika

Browse Source 
Revert "CI: Stress test with praktika"
This commit is contained in:
Max Kainov 2024-11-21 23:09:45 +00:00 committed by GitHub
commit c01720b617
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 79 additions and 743 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
ci/settings/__init__.py → ci/__init__.py
View File

9
ci/docker/fasttest/Dockerfile
View File

@ -102,14 +102,5 @@ RUN groupadd --system --gid 1000 clickhouse \
&& useradd --system --gid 1000 --uid 1000 -m clickhouse \ && useradd --system --gid 1000 --uid 1000 -m clickhouse \
&& mkdir -p /.cache/sccache && chmod 777 /.cache/sccache && mkdir -p /.cache/sccache && chmod 777 /.cache/sccache
# TODO move nfpm to docker that will do packaging
ARG TARGETARCH
ARG NFPM_VERSION=2.20.0
RUN arch=${TARGETARCH:-amd64} \
&& curl -Lo /tmp/nfpm.deb "https://github.com/goreleaser/nfpm/releases/download/v${NFPM_VERSION}/nfpm_${arch}.deb" \
&& dpkg -i /tmp/nfpm.deb \
&& rm /tmp/nfpm.deb
ENV PYTHONPATH="/wd" ENV PYTHONPATH="/wd"
ENV PYTHONUNBUFFERED=1 ENV PYTHONUNBUFFERED=1

2
ci/docker/stateless-test/Dockerfile
View File

@ -58,7 +58,6 @@ RUN apt-get update -y \
curl \ curl \
wget \ wget \
xz-utils \ xz-utils \
ripgrep \
&& apt-get clean \ && apt-get clean \
&& rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/* && rm -rf /var/lib/apt/lists/* /var/cache/debconf /tmp/*
@ -115,5 +114,4 @@ RUN curl -L --no-verbose -O 'https://archive.apache.org/dist/hadoop/common/hadoo
RUN npm install -g azurite@3.30.0 \ RUN npm install -g azurite@3.30.0 \
&& npm install -g tslib && npm install -g node && npm install -g tslib && npm install -g node
ENV PYTHONPATH=".:./ci"
USER clickhouse USER clickhouse

1
ci/docker/stateless-test/requirements.txt
View File

@ -4,4 +4,3 @@ requests==2.32.3
pandas==1.5.3 pandas==1.5.3
scipy==1.12.0 scipy==1.12.0
pyarrow==18.0.0 pyarrow==18.0.0
grpcio==1.47.0

0
ci/jobs/__init__.py Normal file
View File

60
ci/jobs/build_clickhouse.py
View File

@ -4,15 +4,11 @@ from praktika.result import Result
from praktika.settings import Settings from praktika.settings import Settings
from praktika.utils import MetaClasses, Shell, Utils from praktika.utils import MetaClasses, Shell, Utils
from ci.jobs.scripts.clickhouse_version import CHVersion
class JobStages(metaclass=MetaClasses.WithIter): class JobStages(metaclass=MetaClasses.WithIter):
CHECKOUT_SUBMODULES = "checkout" CHECKOUT_SUBMODULES = "checkout"
CMAKE = "cmake" CMAKE = "cmake"
UNSHALLOW = "unshallow"
BUILD = "build" BUILD = "build"
PACKAGE = "package"
def parse_args(): def parse_args():
@ -37,7 +33,8 @@ CMAKE_CMD = """cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA \
-DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \ -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_INSTALL_LOCALSTATEDIR=/var -DCMAKE_SKIP_INSTALL_ALL_DEPENDENCY=ON \
{AUX_DEFS} \ {AUX_DEFS} \
-DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 \ -DCMAKE_C_COMPILER=clang-18 -DCMAKE_CXX_COMPILER=clang++-18 \
-DCOMPILER_CACHE={CACHE_TYPE} -DENABLE_BUILD_PROFILING=1 {DIR}""" -DCOMPILER_CACHE={CACHE_TYPE} \
-DENABLE_BUILD_PROFILING=1 {DIR}"""
def main(): def main():
@ -94,27 +91,6 @@ def main():
res = True res = True
results = [] results = []
version = ""
if res and JobStages.UNSHALLOW in stages:
results.append(
Result.create_from_command_execution(
name="Repo Unshallow",
command="git rev-parse --is-shallow-repository | grep -q true && git fetch --depth 10000 --no-tags --filter=tree:0 origin $(git rev-parse --abbrev-ref HEAD)",
with_log=True,
)
)
res = results[-1].is_ok()
if res:
try:
version = CHVersion.get_version()
assert version
print(f"Got version from repo [{version}]")
except Exception as e:
results[-1].set_failed().set_info(
f"Failed to get version from repo, ex [{e}]"
)
res = False
if res and JobStages.CHECKOUT_SUBMODULES in stages: if res and JobStages.CHECKOUT_SUBMODULES in stages:
Shell.check(f"rm -rf {build_dir} && mkdir -p {build_dir}") Shell.check(f"rm -rf {build_dir} && mkdir -p {build_dir}")
@ -151,38 +127,6 @@ def main():
Shell.check(f"ls -l {build_dir}/programs/") Shell.check(f"ls -l {build_dir}/programs/")
res = results[-1].is_ok() res = results[-1].is_ok()
if res and JobStages.PACKAGE in stages:
if "debug" in build_type:
package_type = "debug"
elif "release" in build_type:
package_type = "release"
elif "asan" in build_type:
package_type = "asan"
else:
assert False, "TODO"
if "amd" in build_type:
deb_arch = "amd64"
else:
deb_arch = "arm64"
output_dir = "/tmp/praktika/output/"
assert Shell.check(f"rm -f {output_dir}/*.deb")
results.append(
Result.create_from_command_execution(
name="Build Packages",
command=[
f"DESTDIR={build_dir}/root ninja programs/install",
f"ln -sf {build_dir}/root {Utils.cwd()}/packages/root",
f"cd {Utils.cwd()}/packages/ && OUTPUT_DIR={output_dir} BUILD_TYPE={package_type} VERSION_STRING={version} DEB_ARCH={deb_arch} ./build --deb",
],
workdir=build_dir,
with_log=True,
)
)
res = results[-1].is_ok()
Result.create_from(results=results, stopwatch=stop_watch).complete_job() Result.create_from(results=results, stopwatch=stop_watch).complete_job()

5
ci/jobs/functional_stateful_tests.py
View File

@ -1,4 +1,5 @@
import argparse import argparse
import os
import time import time
from pathlib import Path from pathlib import Path
@ -130,10 +131,6 @@ def main():
) )
res = res and CH.start() res = res and CH.start()
res = res and CH.wait_ready() res = res and CH.wait_ready()
# TODO: Use --database-replicated optionally
res = res and Shell.check(
f"./ci/jobs/scripts/functional_tests/setup_ch_cluster.sh"
)
if res: if res:
print("ch started") print("ch started")
logs_to_attach += [ logs_to_attach += [

4
ci/jobs/functional_stateless_tests.py
View File

@ -101,7 +101,6 @@ def main():
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client", f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-client",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor", f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-compressor",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local", f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-local",
f"ln -sf {ch_path}/clickhouse {ch_path}/clickhouse-disks",
f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server", f"rm -rf {Settings.TEMP_DIR}/etc/ && mkdir -p {Settings.TEMP_DIR}/etc/clickhouse-client {Settings.TEMP_DIR}/etc/clickhouse-server",
f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/", f"cp programs/server/config.xml programs/server/users.xml {Settings.TEMP_DIR}/etc/clickhouse-server/",
# TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled # TODO: find a way to work with Azure secret so it's ok for local tests as well, for now keep azure disabled
@ -115,7 +114,6 @@ def main():
f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done", f"for file in /tmp/praktika/etc/clickhouse-server/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|>/var/log|>{Settings.TEMP_DIR}/var/log|g; s|>/etc/|>{Settings.TEMP_DIR}/etc/|g' $(readlink -f $file); done",
f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|<path>local_disk|<path>{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done", f"for file in /tmp/praktika/etc/clickhouse-server/config.d/*.xml; do [ -f $file ] && echo Change config $file && sed -i 's|<path>local_disk|<path>{Settings.TEMP_DIR}/local_disk|g' $(readlink -f $file); done",
f"clickhouse-server --version", f"clickhouse-server --version",
f"chmod +x /tmp/praktika/input/clickhouse-odbc-bridge",
] ]
results.append( results.append(
Result.create_from_command_execution( Result.create_from_command_execution(
@ -140,7 +138,6 @@ def main():
res = res and Shell.check( res = res and Shell.check(
"aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True "aws s3 ls s3://test --endpoint-url http://localhost:11111/", verbose=True
) )
res = res and CH.log_cluster_config()
res = res and CH.start() res = res and CH.start()
res = res and CH.wait_ready() res = res and CH.wait_ready()
if res: if res:
@ -173,7 +170,6 @@ def main():
batch_total=total_batches, batch_total=total_batches,
test=args.test, test=args.test,
) )
CH.log_cluster_stop_replication()
results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run()) results.append(FTResultsProcessor(wd=Settings.OUTPUT_DIR).run())
results[-1].set_timing(stopwatch=stop_watch_) results[-1].set_timing(stopwatch=stop_watch_)
res = results[-1].is_ok() res = results[-1].is_ok()

18
ci/jobs/scripts/clickhouse_proc.py
View File

@ -66,24 +66,6 @@ class ClickHouseProc:
print(f"Started setup_minio.sh asynchronously with PID {process.pid}") print(f"Started setup_minio.sh asynchronously with PID {process.pid}")
return True return True
def log_cluster_config(self):
return Shell.check(
f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --config-logs-export-cluster /tmp/praktika/etc/clickhouse-server/config.d/system_logs_export.yaml",
verbose=True,
)
def log_cluster_setup_replication(self):
return Shell.check(
f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --setup-logs-replication",
verbose=True,
)
def log_cluster_stop_replication(self):
return Shell.check(
f"./ci/jobs/scripts/functional_tests/setup_log_cluster.sh --stop-log-replication",
verbose=True,
)
def start(self): def start(self):
print("Starting ClickHouse server") print("Starting ClickHouse server")
Shell.check(f"rm {self.pid_file}") Shell.check(f"rm {self.pid_file}")

38
ci/jobs/scripts/clickhouse_version.py
View File

@ -1,38 +0,0 @@
from pathlib import Path
from praktika.utils import Shell
class CHVersion:
FILE_WITH_VERSION_PATH = "./cmake/autogenerated_versions.txt"
@classmethod
def _get_tweak(cls):
tag = Shell.get_output("git describe --tags --abbrev=0")
assert tag.startswith("v24")
num = Shell.get_output(f"git rev-list --count {tag}..HEAD")
return int(num)
@classmethod
def get_version(cls):
versions = {}
for line in (
Path(cls.FILE_WITH_VERSION_PATH).read_text(encoding="utf-8").splitlines()
):
line = line.strip()
if not line.startswith("SET("):
continue
name, value = line[4:-1].split(maxsplit=1)
name = name.removeprefix("VERSION_").lower()
try:
value = int(value)
except ValueError:
pass
versions[name] = value
version_sha = versions["githash"]
tweak = int(
Shell.get_output(f"git rev-list --count {version_sha}..HEAD", verbose=True)
)
return f"{versions['major']}.{versions['minor']}.{versions['patch']}.{tweak}"

118
ci/jobs/scripts/functional_tests/setup_ch_cluster.sh
View File

@ -1,118 +0,0 @@
#!/bin/bash
set -e -x
clickhouse-client --query "SHOW DATABASES"
clickhouse-client --query "CREATE DATABASE datasets"
clickhouse-client < ./tests/docker_scripts/create.sql
clickhouse-client --query "SHOW TABLES FROM datasets"
USE_DATABASE_REPLICATED=0
while [[ "$#" -gt 0 ]]; do
case $1 in
--database-replicated)
echo "Setup cluster for testing with Database Replicated"
USE_DATABASE_REPLICATED=1
;;
*)
echo "Unknown option: $1"
exit 1
;;
esac
shift
done
if [[ "$USE_DATABASE_REPLICATED" -eq 1 ]]; then
clickhouse-client --query "CREATE DATABASE test ON CLUSTER 'test_cluster_database_replicated'
ENGINE=Replicated('/test/clickhouse/db/test', '{shard}', '{replica}')"
clickhouse-client --query "CREATE TABLE test.hits AS datasets.hits_v1"
clickhouse-client --query "CREATE TABLE test.visits AS datasets.visits_v1"
clickhouse-client --max_memory_usage 10G --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1"
clickhouse-client --max_memory_usage 10G --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1"
clickhouse-client --query "DROP TABLE datasets.hits_v1"
clickhouse-client --query "DROP TABLE datasets.visits_v1"
else
clickhouse-client --query "CREATE DATABASE test"
clickhouse-client --query "SHOW TABLES FROM test"
if [[ -n "$USE_S3_STORAGE_FOR_MERGE_TREE" ]] && [[ "$USE_S3_STORAGE_FOR_MERGE_TREE" -eq 1 ]]; then
clickhouse-client --query "CREATE TABLE test.hits (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16,
EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32,
UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String,
RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16),
URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8,
FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16,
UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8,
MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16,
SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16,
ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32,
SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8,
FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8,
HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8,
GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32,
HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String,
HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32,
FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32,
LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32,
RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String,
ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String,
OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String,
UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64,
URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String,
ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate)
ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
clickhouse-client --query "CREATE TABLE test.visits (CounterID UInt32, StartDate Date, Sign Int8, IsNew UInt8,
VisitID UInt64, UserID UInt64, StartTime DateTime, Duration UInt32, UTCStartTime DateTime, PageViews Int32,
Hits Int32, IsBounce UInt8, Referer String, StartURL String, RefererDomain String, StartURLDomain String,
EndURL String, LinkURL String, IsDownload UInt8, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String,
AdvEngineID UInt8, PlaceID Int32, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32),
RefererRegions Array(UInt32), IsYandex UInt8, GoalReachesDepth Int32, GoalReachesURL Int32, GoalReachesAny Int32,
SocialSourceNetworkID UInt8, SocialSourcePage String, MobilePhoneModel String, ClientEventTime DateTime, RegionID UInt32,
ClientIP UInt32, ClientIP6 FixedString(16), RemoteIP UInt32, RemoteIP6 FixedString(16), IPNetworkID UInt32,
SilverlightVersion3 UInt32, CodeVersion UInt32, ResolutionWidth UInt16, ResolutionHeight UInt16, UserAgentMajor UInt16,
UserAgentMinor UInt16, WindowClientWidth UInt16, WindowClientHeight UInt16, SilverlightVersion2 UInt8, SilverlightVersion4 UInt16,
FlashVersion3 UInt16, FlashVersion4 UInt16, ClientTimeZone Int16, OS UInt8, UserAgent UInt8, ResolutionDepth UInt8,
FlashMajor UInt8, FlashMinor UInt8, NetMajor UInt8, NetMinor UInt8, MobilePhone UInt8, SilverlightVersion1 UInt8,
Age UInt8, Sex UInt8, Income UInt8, JavaEnable UInt8, CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8,
BrowserLanguage UInt16, BrowserCountry UInt16, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16),
Params Array(String), Goals Nested(ID UInt32, Serial UInt32, EventTime DateTime, Price Int64, OrderID String, CurrencyID UInt32),
WatchIDs Array(UInt64), ParamSumPrice Int64, ParamCurrency FixedString(3), ParamCurrencyID UInt16, ClickLogID UInt64,
ClickEventID Int32, ClickGoodEvent Int32, ClickEventTime DateTime, ClickPriorityID Int32, ClickPhraseID Int32, ClickPageID Int32,
ClickPlaceID Int32, ClickTypeID Int32, ClickResourceID Int32, ClickCost UInt32, ClickClientIP UInt32, ClickDomainID UInt32,
ClickURL String, ClickAttempt UInt8, ClickOrderID UInt32, ClickBannerID UInt32, ClickMarketCategoryID UInt32, ClickMarketPP UInt32,
ClickMarketCategoryName String, ClickMarketPPName String, ClickAWAPSCampaignName String, ClickPageName String, ClickTargetType UInt16,
ClickTargetPhraseID UInt64, ClickContextType UInt8, ClickSelectType Int8, ClickOptions String, ClickGroupBannerID Int32,
OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String,
UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, FirstVisit DateTime,
PredLastVisit Date, LastVisit Date, TotalVisits UInt32, TraficSource Nested(ID Int8, SearchEngineID UInt16, AdvEngineID UInt8,
PlaceID UInt16, SocialSourceNetworkID UInt8, Domain String, SearchPhrase String, SocialSourcePage String), Attendance FixedString(16),
CLID UInt32, YCLID UInt64, NormalizedRefererHash UInt64, SearchPhraseHash UInt64, RefererDomainHash UInt64, NormalizedStartURLHash UInt64,
StartURLDomainHash UInt64, NormalizedEndURLHash UInt64, TopLevelDomain UInt64, URLScheme UInt64, OpenstatServiceNameHash UInt64,
OpenstatCampaignIDHash UInt64, OpenstatAdIDHash UInt64, OpenstatSourceIDHash UInt64, UTMSourceHash UInt64, UTMMediumHash UInt64,
UTMCampaignHash UInt64, UTMContentHash UInt64, UTMTermHash UInt64, FromHash UInt64, WebVisorEnabled UInt8, WebVisorActivity UInt32,
ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64),
Market Nested(Type UInt8, GoalID UInt32, OrderID String, OrderPrice Int64, PP UInt32, DirectPlaceID UInt32, DirectOrderID UInt32,
DirectBannerID UInt32, GoodID String, GoodName String, GoodQuantity Int32, GoodPrice Int64), IslandID FixedString(16))
ENGINE = CollapsingMergeTree(Sign) PARTITION BY toYYYYMM(StartDate) ORDER BY (CounterID, StartDate, intHash32(UserID), VisitID)
SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
clickhouse-client --max_memory_usage 10G --query "INSERT INTO test.hits SELECT * FROM datasets.hits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
clickhouse-client --max_memory_usage 10G --query "INSERT INTO test.visits SELECT * FROM datasets.visits_v1 SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
clickhouse-client --query "DROP TABLE datasets.visits_v1 SYNC"
clickhouse-client --query "DROP TABLE datasets.hits_v1 SYNC"
else
clickhouse-client --query "RENAME TABLE datasets.hits_v1 TO test.hits"
clickhouse-client --query "RENAME TABLE datasets.visits_v1 TO test.visits"
fi
clickhouse-client --query "CREATE TABLE test.hits_s3 (WatchID UInt64, JavaEnable UInt8, Title String, GoodEvent Int16, EventTime DateTime, EventDate Date, CounterID UInt32, ClientIP UInt32, ClientIP6 FixedString(16), RegionID UInt32, UserID UInt64, CounterClass Int8, OS UInt8, UserAgent UInt8, URL String, Referer String, URLDomain String, RefererDomain String, Refresh UInt8, IsRobot UInt8, RefererCategories Array(UInt16), URLCategories Array(UInt16), URLRegions Array(UInt32), RefererRegions Array(UInt32), ResolutionWidth UInt16, ResolutionHeight UInt16, ResolutionDepth UInt8, FlashMajor UInt8, FlashMinor UInt8, FlashMinor2 String, NetMajor UInt8, NetMinor UInt8, UserAgentMajor UInt16, UserAgentMinor FixedString(2), CookieEnable UInt8, JavascriptEnable UInt8, IsMobile UInt8, MobilePhone UInt8, MobilePhoneModel String, Params String, IPNetworkID UInt32, TraficSourceID Int8, SearchEngineID UInt16, SearchPhrase String, AdvEngineID UInt8, IsArtifical UInt8, WindowClientWidth UInt16, WindowClientHeight UInt16, ClientTimeZone Int16, ClientEventTime DateTime, SilverlightVersion1 UInt8, SilverlightVersion2 UInt8, SilverlightVersion3 UInt32, SilverlightVersion4 UInt16, PageCharset String, CodeVersion UInt32, IsLink UInt8, IsDownload UInt8, IsNotBounce UInt8, FUniqID UInt64, HID UInt32, IsOldCounter UInt8, IsEvent UInt8, IsParameter UInt8, DontCountHits UInt8, WithHash UInt8, HitColor FixedString(1), UTCEventTime DateTime, Age UInt8, Sex UInt8, Income UInt8, Interests UInt16, Robotness UInt8, GeneralInterests Array(UInt16), RemoteIP UInt32, RemoteIP6 FixedString(16), WindowName Int32, OpenerName Int32, HistoryLength Int16, BrowserLanguage FixedString(2), BrowserCountry FixedString(2), SocialNetwork String, SocialAction String, HTTPError UInt16, SendTiming Int32, DNSTiming Int32, ConnectTiming Int32, ResponseStartTiming Int32, ResponseEndTiming Int32, FetchTiming Int32, RedirectTiming Int32, DOMInteractiveTiming Int32, DOMContentLoadedTiming Int32, DOMCompleteTiming Int32, LoadEventStartTiming Int32, LoadEventEndTiming Int32, NSToDOMContentLoadedTiming Int32, FirstPaintTiming Int32, RedirectCount Int8, SocialSourceNetworkID UInt8, SocialSourcePage String, ParamPrice Int64, ParamOrderID String, ParamCurrency FixedString(3), ParamCurrencyID UInt16, GoalsReached Array(UInt32), OpenstatServiceName String, OpenstatCampaignID String, OpenstatAdID String, OpenstatSourceID String, UTMSource String, UTMMedium String, UTMCampaign String, UTMContent String, UTMTerm String, FromTag String, HasGCLID UInt8, RefererHash UInt64, URLHash UInt64, CLID UInt32, YCLID UInt64, ShareService String, ShareURL String, ShareTitle String, ParsedParams Nested(Key1 String, Key2 String, Key3 String, Key4 String, Key5 String, ValueDouble Float64), IslandID FixedString(16), RequestNum UInt32, RequestTry UInt8) ENGINE = MergeTree() PARTITION BY toYYYYMM(EventDate) ORDER BY (CounterID, EventDate, intHash32(UserID)) SAMPLE BY intHash32(UserID) SETTINGS index_granularity = 8192, storage_policy='s3_cache'"
# AWS S3 is very inefficient, so increase memory even further:
clickhouse-client --max_memory_usage 30G --max_memory_usage_for_user 30G --query "INSERT INTO test.hits_s3 SELECT * FROM test.hits SETTINGS enable_filesystem_cache_on_write_operations=0, max_insert_threads=16"
fi
clickhouse-client --query "SHOW TABLES FROM test"
clickhouse-client --query "SELECT count() FROM test.hits"
clickhouse-client --query "SELECT count() FROM test.visits"

261
ci/jobs/scripts/functional_tests/setup_log_cluster.sh
View File

@ -1,261 +0,0 @@
#!/bin/bash
set -e
# This script sets up export of system log tables to a remote server.
# Remote tables are created if not exist, and augmented with extra columns,
# and their names will contain a hash of the table structure,
# which allows exporting tables from servers of different versions.
# Config file contains KEY=VALUE pairs with any necessary parameters like:
# CLICKHOUSE_CI_LOGS_HOST - remote host
# CLICKHOUSE_CI_LOGS_USER - password for user
# CLICKHOUSE_CI_LOGS_PASSWORD - password for user
CLICKHOUSE_CI_LOGS_CREDENTIALS=${CLICKHOUSE_CI_LOGS_CREDENTIALS:-/tmp/export-logs-config.sh}
CLICKHOUSE_CI_LOGS_USER=${CLICKHOUSE_CI_LOGS_USER:-ci}
# Pre-configured destination cluster, where to export the data
CLICKHOUSE_CI_LOGS_CLUSTER=${CLICKHOUSE_CI_LOGS_CLUSTER:-system_logs_export}
EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime('UTC'), check_name LowCardinality(String), instance_type LowCardinality(String), instance_id String, INDEX ix_pr (pull_request_number) TYPE set(100), INDEX ix_commit (commit_sha) TYPE set(100), INDEX ix_check_time (check_start_time) TYPE minmax, "}
EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"CAST(0 AS UInt32) AS pull_request_number, '' AS commit_sha, now() AS check_start_time, toLowCardinality('') AS check_name, toLowCardinality('') AS instance_type, '' AS instance_id"}
EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:-"check_name"}
# trace_log needs more columns for symbolization
EXTRA_COLUMNS_TRACE_LOG="${EXTRA_COLUMNS} symbols Array(LowCardinality(String)), lines Array(LowCardinality(String)), "
EXTRA_COLUMNS_EXPRESSION_TRACE_LOG="${EXTRA_COLUMNS_EXPRESSION}, arrayMap(x -> demangle(addressToSymbol(x)), trace)::Array(LowCardinality(String)) AS symbols, arrayMap(x -> addressToLine(x), trace)::Array(LowCardinality(String)) AS lines"
# coverage_log needs more columns for symbolization, but only symbol names (the line numbers are too heavy to calculate)
EXTRA_COLUMNS_COVERAGE_LOG="${EXTRA_COLUMNS} symbols Array(LowCardinality(String)), "
EXTRA_COLUMNS_EXPRESSION_COVERAGE_LOG="${EXTRA_COLUMNS_EXPRESSION}, arrayDistinct(arrayMap(x -> demangle(addressToSymbol(x)), coverage))::Array(LowCardinality(String)) AS symbols"
function __set_connection_args
{
# It's impossible to use a generic $CONNECTION_ARGS string, it's unsafe from word splitting perspective.
# That's why we must stick to the generated option
CONNECTION_ARGS=(
--receive_timeout=45 --send_timeout=45 --secure
--user "${CLICKHOUSE_CI_LOGS_USER}" --host "${CLICKHOUSE_CI_LOGS_HOST}"
--password "${CLICKHOUSE_CI_LOGS_PASSWORD}"
)
}
function __shadow_credentials
{
# The function completely screws the output, it shouldn't be used in normal functions, only in ()
# The only way to substitute the env as a plain text is using perl 's/\Qsomething\E/another/
exec &> >(perl -pe '
s(\Q$ENV{CLICKHOUSE_CI_LOGS_HOST}\E)[CLICKHOUSE_CI_LOGS_HOST]g;
s(\Q$ENV{CLICKHOUSE_CI_LOGS_USER}\E)[CLICKHOUSE_CI_LOGS_USER]g;
s(\Q$ENV{CLICKHOUSE_CI_LOGS_PASSWORD}\E)[CLICKHOUSE_CI_LOGS_PASSWORD]g;
')
}
function check_logs_credentials
(
# The function connects with given credentials, and if it's unable to execute the simplest query, returns exit code
# First check, if all necessary parameters are set
set +x
for parameter in CLICKHOUSE_CI_LOGS_HOST CLICKHOUSE_CI_LOGS_USER CLICKHOUSE_CI_LOGS_PASSWORD; do
export -p | grep -q "$parameter" || {
echo "Credentials parameter $parameter is unset"
return 1
}
done
__shadow_credentials
__set_connection_args
local code
# Catch both success and error to not fail on `set -e`
clickhouse-client "${CONNECTION_ARGS[@]}" -q 'SELECT 1 FORMAT Null' && return 0 || code=$?
if [ "$code" != 0 ]; then
echo 'Failed to connect to CI Logs cluster'
return $code
fi
)
function config_logs_export_cluster
(
# The function is launched in a separate shell instance to not expose the
# exported values from CLICKHOUSE_CI_LOGS_CREDENTIALS
set +x
if ! [ -r "${CLICKHOUSE_CI_LOGS_CREDENTIALS}" ]; then
echo "File $CLICKHOUSE_CI_LOGS_CREDENTIALS does not exist, do not setup"
return
fi
set -a
# shellcheck disable=SC1090
source "${CLICKHOUSE_CI_LOGS_CREDENTIALS}"
set +a
__shadow_credentials
echo "Checking if the credentials work"
check_logs_credentials || return 0
cluster_config="${1:-/etc/clickhouse-server/config.d/system_logs_export.yaml}"
mkdir -p "$(dirname "$cluster_config")"
echo "remote_servers:
${CLICKHOUSE_CI_LOGS_CLUSTER}:
shard:
replica:
secure: 1
user: '${CLICKHOUSE_CI_LOGS_USER}'
host: '${CLICKHOUSE_CI_LOGS_HOST}'
port: 9440
password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
" > "$cluster_config"
echo "Cluster ${CLICKHOUSE_CI_LOGS_CLUSTER} is confugured in ${cluster_config}"
)
function setup_logs_replication
(
# The function is launched in a separate shell instance to not expose the
# exported values from CLICKHOUSE_CI_LOGS_CREDENTIALS
set +x
# disable output
if ! [ -r "${CLICKHOUSE_CI_LOGS_CREDENTIALS}" ]; then
echo "File $CLICKHOUSE_CI_LOGS_CREDENTIALS does not exist, do not setup"
return 0
fi
set -a
# shellcheck disable=SC1090
source "${CLICKHOUSE_CI_LOGS_CREDENTIALS}"
set +a
__shadow_credentials
echo "Checking if the credentials work"
check_logs_credentials || return 0
__set_connection_args
echo "My hostname is ${HOSTNAME}"
echo 'Create all configured system logs'
clickhouse-client --query "SYSTEM FLUSH LOGS"
debug_or_sanitizer_build=$(clickhouse-client -q "WITH ((SELECT value FROM system.build_options WHERE name='BUILD_TYPE') AS build, (SELECT value FROM system.build_options WHERE name='CXX_FLAGS') as flags) SELECT build='Debug' OR flags LIKE '%fsanitize%'")
echo "Build is debug or sanitizer: $debug_or_sanitizer_build"
# We will pre-create a table system.coverage_log.
# It is normally created by clickhouse-test rather than the server,
# so we will create it in advance to make it be picked up by the next commands:
clickhouse-client --query "
CREATE TABLE IF NOT EXISTS system.coverage_log
(
time DateTime COMMENT 'The time of test run',
test_name String COMMENT 'The name of the test',
coverage Array(UInt64) COMMENT 'An array of addresses of the code (a subset of addresses instrumented for coverage) that were encountered during the test run'
) ENGINE = MergeTree ORDER BY test_name COMMENT 'Contains information about per-test coverage from the CI, but used only for exporting to the CI cluster'
"
# For each system log table:
echo 'Create %_log tables'
clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
do
if [[ "$table" = "trace_log" ]]
then
EXTRA_COLUMNS_FOR_TABLE="${EXTRA_COLUMNS_TRACE_LOG}"
# Do not try to resolve stack traces in case of debug/sanitizers
# build, since it is too slow (flushing of trace_log can take ~1min
# with such MV attached)
if [[ "$debug_or_sanitizer_build" = 1 ]]
then
EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION}"
else
EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION_TRACE_LOG}"
fi
elif [[ "$table" = "coverage_log" ]]
then
EXTRA_COLUMNS_FOR_TABLE="${EXTRA_COLUMNS_COVERAGE_LOG}"
EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION_COVERAGE_LOG}"
else
EXTRA_COLUMNS_FOR_TABLE="${EXTRA_COLUMNS}"
EXTRA_COLUMNS_EXPRESSION_FOR_TABLE="${EXTRA_COLUMNS_EXPRESSION}"
fi
# Calculate hash of its structure. Note: 4 is the version of extra columns - increment it if extra columns are changed:
hash=$(clickhouse-client --query "
SELECT sipHash64(9, groupArray((name, type)))
FROM (SELECT name, type FROM system.columns
WHERE database = 'system' AND table = '$table'
ORDER BY position)
")
# Create the destination table with adapted name and structure:
statement=$(clickhouse-client --format TSVRaw --query "SHOW CREATE TABLE system.${table}" | sed -r -e '
s/^\($/('"$EXTRA_COLUMNS_FOR_TABLE"'/;
s/^ORDER BY (([^\(].+?)|\((.+?)\))$/ORDER BY ('"$EXTRA_ORDER_BY_COLUMNS"', \2\3)/;
s/^CREATE TABLE system\.\w+_log$/CREATE TABLE IF NOT EXISTS '"$table"'_'"$hash"'/;
/^TTL /d
')
echo -e "Creating remote destination table ${table}_${hash} with statement:" >&2
echo "::group::${table}"
# there's the only way big "$statement" can be printed without causing EAGAIN error
# cat: write error: Resource temporarily unavailable
statement_print="${statement}"
if [ "${#statement_print}" -gt 4000 ]; then
statement_print="${statement::1999}\n…\n${statement:${#statement}-1999}"
fi
echo -e "$statement_print"
echo "::endgroup::"
echo "$statement" | clickhouse-client --database_replicated_initial_query_timeout_sec=10 \
--distributed_ddl_task_timeout=30 --distributed_ddl_output_mode=throw_only_active \
"${CONNECTION_ARGS[@]}" || continue
echo "Creating table system.${table}_sender" >&2
# Create Distributed table and materialized view to watch on the original table:
clickhouse-client --query "
CREATE TABLE system.${table}_sender
ENGINE = Distributed(${CLICKHOUSE_CI_LOGS_CLUSTER}, default, ${table}_${hash})
SETTINGS flush_on_detach=0
EMPTY AS
SELECT ${EXTRA_COLUMNS_EXPRESSION_FOR_TABLE}, *
FROM system.${table}
" || continue
echo "Creating materialized view system.${table}_watcher" >&2
clickhouse-client --query "
CREATE MATERIALIZED VIEW system.${table}_watcher TO system.${table}_sender AS
SELECT ${EXTRA_COLUMNS_EXPRESSION_FOR_TABLE}, *
FROM system.${table}
" || continue
done
)
function stop_logs_replication
{
echo "Detach all logs replication"
clickhouse-client --query "select database||'.'||table from system.tables where database = 'system' and (table like '%_sender' or table like '%_watcher')" | {
tee /dev/stderr
} | {
timeout --preserve-status --signal TERM --kill-after 5m 15m xargs -n1 -r -i clickhouse-client --query "drop table {}"
}
}
while [[ "$#" -gt 0 ]]; do
case $1 in
--stop-log-replication)
echo "Stopping log replication..."
stop_logs_replication
;;
--setup-logs-replication)
echo "Setting up log replication..."
setup_logs_replication
;;
--config-logs-export-cluster)
echo "Configuring logs export for the cluster..."
config_logs_export_cluster "$2"
shift
;;
*)
echo "Unknown option: $1"
echo "Usage: $0 [--stop-log-replication | --setup-logs-replication | --config-logs-export-cluster ]"
exit 1
;;
esac
shift
done

2
ci/praktika/_environment.py
View File

@ -179,7 +179,7 @@ class _Environment(MetaClasses.Serializable):
if bucket in path: if bucket in path:
path = path.replace(bucket, endpoint) path = path.replace(bucket, endpoint)
break break
REPORT_URL = f"https://{path}/{Path(settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={'latest' if latest else self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}" REPORT_URL = f"https://{path}/{Path(settings.HTML_PAGE_FILE).name}?PR={self.PR_NUMBER}&sha={'latest' if latest else self.SHA}&name_0={urllib.parse.quote(self.WORKFLOW_NAME, safe='')}&name_1={urllib.parse.quote(self.JOB_NAME, safe='')}"
return REPORT_URL return REPORT_URL
def is_local_run(self): def is_local_run(self):

9
ci/praktika/artifact.py
View File

@ -1,4 +1,3 @@
import copy
from dataclasses import dataclass from dataclasses import dataclass
@ -25,14 +24,6 @@ class Artifact:
def is_s3_artifact(self): def is_s3_artifact(self):
return self.type == Artifact.Type.S3 return self.type == Artifact.Type.S3
def parametrize(self, names):
res = []
for name in names:
obj = copy.deepcopy(self)
obj.name = name
res.append(obj)
return res
@classmethod @classmethod
def define_artifact(cls, name, type, path): def define_artifact(cls, name, type, path):
return cls.Config(name=name, type=type, path=path) return cls.Config(name=name, type=type, path=path)

7
ci/praktika/hook_html.py
View File

@ -128,9 +128,6 @@ class HtmlRunnerHooks:
for job in _workflow.jobs: for job in _workflow.jobs:
if job.name not in skip_jobs: if job.name not in skip_jobs:
result = Result.generate_pending(job.name) result = Result.generate_pending(job.name)
# Preemptively add the general job log to the result directory to ensure
# the post-job handler can upload it, even if the job is terminated unexpectedly
result.set_files([Settings.RUN_LOG])
else: else:
result = Result.generate_skipped(job.name, job_cache_records[job.name]) result = Result.generate_skipped(job.name, job_cache_records[job.name])
results.append(result) results.append(result)
@ -140,14 +137,14 @@ class HtmlRunnerHooks:
summary_result.start_time = Utils.timestamp() summary_result.start_time = Utils.timestamp()
assert _ResultS3.copy_result_to_s3_with_version(summary_result, version=0) assert _ResultS3.copy_result_to_s3_with_version(summary_result, version=0)
page_url = env.get_report_url(settings=Settings, latest=True) page_url = env.get_report_url(settings=Settings)
print(f"CI Status page url [{page_url}]") print(f"CI Status page url [{page_url}]")
res1 = GH.post_commit_status( res1 = GH.post_commit_status(
name=_workflow.name, name=_workflow.name,
status=Result.Status.PENDING, status=Result.Status.PENDING,
description="", description="",
url=page_url, url=env.get_report_url(settings=Settings, latest=True),
) )
res2 = GH.post_pr_comment( res2 = GH.post_pr_comment(
comment_body=f"Workflow [[{_workflow.name}]({page_url})], commit [{_Environment.get().SHA[:8]}]", comment_body=f"Workflow [[{_workflow.name}]({page_url})], commit [{_Environment.get().SHA[:8]}]",

2
ci/praktika/json.html
View File

@ -601,7 +601,7 @@
td.classList.add('time-column'); td.classList.add('time-column');
td.textContent = value ? formatDuration(value) : ''; td.textContent = value ? formatDuration(value) : '';
} else if (column === 'info') { } else if (column === 'info') {
td.textContent = value && value.includes('\n') ? '↵' : (value || ''); td.textContent = value.includes('\n') ? '↵' : (value || '');
td.classList.add('info-column'); td.classList.add('info-column');
} }

8
ci/praktika/native_jobs.py
View File

@ -310,7 +310,7 @@ def _finish_workflow(workflow, job_name):
print(env.get_needs_statuses()) print(env.get_needs_statuses())
print("Check Workflow results") print("Check Workflow results")
version = _ResultS3.copy_result_from_s3_with_version( _ResultS3.copy_result_from_s3(
Result.file_name_static(workflow.name), Result.file_name_static(workflow.name),
) )
workflow_result = Result.from_fs(workflow.name) workflow_result = Result.from_fs(workflow.name)
@ -333,7 +333,7 @@ def _finish_workflow(workflow, job_name):
# dump workflow result after update - to have an updated result in post # dump workflow result after update - to have an updated result in post
workflow_result.dump() workflow_result.dump()
# add error into env - should apper in the report # add error into env - should apper in the report
env.add_info(f"{result.name}: {ResultInfo.NOT_FINALIZED}") env.add_info(ResultInfo.NOT_FINALIZED + f" [{result.name}]")
update_final_report = True update_final_report = True
job = workflow.get_job(result.name) job = workflow.get_job(result.name)
if not job or not job.allow_merge_on_failure: if not job or not job.allow_merge_on_failure:
@ -358,7 +358,9 @@ def _finish_workflow(workflow, job_name):
env.add_info(ResultInfo.GH_STATUS_ERROR) env.add_info(ResultInfo.GH_STATUS_ERROR)
if update_final_report: if update_final_report:
_ResultS3.copy_result_to_s3_with_version(workflow_result, version + 1) _ResultS3.copy_result_to_s3(
workflow_result,
)
Result.from_fs(job_name).set_status(Result.Status.SUCCESS) Result.from_fs(job_name).set_status(Result.Status.SUCCESS)

3
ci/praktika/result.py
View File

@ -121,9 +121,6 @@ class Result(MetaClasses.Serializable):
def set_success(self) -> "Result": def set_success(self) -> "Result":
return self.set_status(Result.Status.SUCCESS) return self.set_status(Result.Status.SUCCESS)
def set_failed(self) -> "Result":
return self.set_status(Result.Status.FAILED)
def set_results(self, results: List["Result"]) -> "Result": def set_results(self, results: List["Result"]) -> "Result":
self.results = results self.results = results
self.dump() self.dump()

37
ci/praktika/runner.py
View File

@ -1,5 +1,3 @@
import glob
import json
import os import os
import re import re
import sys import sys
@ -60,9 +58,6 @@ class Runner:
workflow_config.digest_dockers[docker.name] = Digest().calc_docker_digest( workflow_config.digest_dockers[docker.name] = Digest().calc_docker_digest(
docker, workflow.dockers docker, workflow.dockers
) )
# work around for old clickhouse jobs
os.environ["DOCKER_TAG"] = json.dumps(workflow_config.digest_dockers)
workflow_config.dump() workflow_config.dump()
Result.generate_pending(job.name).dump() Result.generate_pending(job.name).dump()
@ -124,21 +119,8 @@ class Runner:
else: else:
prefixes = [env.get_s3_prefix()] * len(required_artifacts) prefixes = [env.get_s3_prefix()] * len(required_artifacts)
for artifact, prefix in zip(required_artifacts, prefixes): for artifact, prefix in zip(required_artifacts, prefixes):
recursive = False s3_path = f"{Settings.S3_ARTIFACT_PATH}/{prefix}/{Utils.normalize_string(artifact._provided_by)}/{Path(artifact.path).name}"
include_pattern = "" assert S3.copy_file_from_s3(s3_path=s3_path, local_path=Settings.INPUT_DIR)
if "*" in artifact.path:
s3_path = f"{Settings.S3_ARTIFACT_PATH}/{prefix}/{Utils.normalize_string(artifact._provided_by)}/"
recursive = True
include_pattern = Path(artifact.path).name
assert "*" in include_pattern
else:
s3_path = f"{Settings.S3_ARTIFACT_PATH}/{prefix}/{Utils.normalize_string(artifact._provided_by)}/{Path(artifact.path).name}"
assert S3.copy_file_from_s3(
s3_path=s3_path,
local_path=Settings.INPUT_DIR,
recursive=recursive,
include_pattern=include_pattern,
)
return 0 return 0
@ -257,11 +239,9 @@ class Runner:
info = f"ERROR: {ResultInfo.KILLED}" info = f"ERROR: {ResultInfo.KILLED}"
print(info) print(info)
result.set_info(info).set_status(Result.Status.ERROR).dump() result.set_info(info).set_status(Result.Status.ERROR).dump()
else:
# TODO: add setting with different ways of storing general praktika log: always, on error, never.
# now let's store it on error only
result.files = [file for file in result.files if file != Settings.RUN_LOG]
if not result.is_ok():
result.set_files(files=[Settings.RUN_LOG])
result.update_duration().dump() result.update_duration().dump()
if run_exit_code == 0: if run_exit_code == 0:
@ -282,11 +262,10 @@ class Runner:
f"ls -l {artifact.path}", verbose=True f"ls -l {artifact.path}", verbose=True
), f"Artifact {artifact.path} not found" ), f"Artifact {artifact.path} not found"
s3_path = f"{Settings.S3_ARTIFACT_PATH}/{env.get_s3_prefix()}/{Utils.normalize_string(env.JOB_NAME)}" s3_path = f"{Settings.S3_ARTIFACT_PATH}/{env.get_s3_prefix()}/{Utils.normalize_string(env.JOB_NAME)}"
for file_path in glob.glob(artifact.path): link = S3.copy_file_to_s3(
link = S3.copy_file_to_s3( s3_path=s3_path, local_path=artifact.path
s3_path=s3_path, local_path=file_path )
) result.set_link(link)
result.set_link(link)
except Exception as e: except Exception as e:
error = ( error = (
f"ERROR: Failed to upload artifact [{artifact}], ex [{e}]" f"ERROR: Failed to upload artifact [{artifact}], ex [{e}]"

13
ci/praktika/s3.py
View File

@ -2,7 +2,6 @@ import dataclasses
import json import json
from pathlib import Path from pathlib import Path
from typing import Dict from typing import Dict
from urllib.parse import quote
from praktika._environment import _Environment from praktika._environment import _Environment
from praktika.settings import Settings from praktika.settings import Settings
@ -56,7 +55,7 @@ class S3:
bucket = s3_path.split("/")[0] bucket = s3_path.split("/")[0]
endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket] endpoint = Settings.S3_BUCKET_TO_HTTP_ENDPOINT[bucket]
assert endpoint assert endpoint
return quote(f"https://{s3_full_path}".replace(bucket, endpoint), safe=":/?&=") return f"https://{s3_full_path}".replace(bucket, endpoint)
@classmethod @classmethod
def put(cls, s3_path, local_path, text=False, metadata=None, if_none_matched=False): def put(cls, s3_path, local_path, text=False, metadata=None, if_none_matched=False):
@ -118,21 +117,15 @@ class S3:
return res return res
@classmethod @classmethod
def copy_file_from_s3( def copy_file_from_s3(cls, s3_path, local_path):
cls, s3_path, local_path, recursive=False, include_pattern=""
):
assert Path(s3_path), f"Invalid S3 Path [{s3_path}]" assert Path(s3_path), f"Invalid S3 Path [{s3_path}]"
if Path(local_path).is_dir(): if Path(local_path).is_dir():
pass local_path = Path(local_path) / Path(s3_path).name
else: else:
assert Path( assert Path(
local_path local_path
).parent.is_dir(), f"Parent path for [{local_path}] does not exist" ).parent.is_dir(), f"Parent path for [{local_path}] does not exist"
cmd = f"aws s3 cp s3://{s3_path} {local_path}" cmd = f"aws s3 cp s3://{s3_path} {local_path}"
if recursive:
cmd += " --recursive"
if include_pattern:
cmd += f" --include {include_pattern}"
res = cls.run_command_with_retries(cmd) res = cls.run_command_with_retries(cmd)
return res return res

1
ci/settings/definitions.py
View File

@ -242,4 +242,3 @@ class JobNames:
BUILD = "Build" BUILD = "Build"
STATELESS = "Stateless tests" STATELESS = "Stateless tests"
STATEFUL = "Stateful tests" STATEFUL = "Stateful tests"
STRESS = "Stress tests"

114
ci/workflows/pull_request.py
View File

@ -16,16 +16,6 @@ class ArtifactNames:
CH_ARM_RELEASE = "CH_ARM_RELEASE" CH_ARM_RELEASE = "CH_ARM_RELEASE"
CH_ARM_ASAN = "CH_ARM_ASAN" CH_ARM_ASAN = "CH_ARM_ASAN"
CH_ODBC_B_AMD_DEBUG = "CH_ODBC_B_AMD_DEBUG"
CH_ODBC_B_AMD_RELEASE = "CH_ODBC_B_AMD_RELEASE"
CH_ODBC_B_ARM_RELEASE = "CH_ODBC_B_ARM_RELEASE"
CH_ODBC_B_ARM_ASAN = "CH_ODBC_B_ARM_ASAN"
DEB_AMD_DEBUG = "DEB_AMD_DEBUG"
DEB_AMD_RELEASE = "DEB_AMD_RELEASE"
DEB_ARM_RELEASE = "DEB_ARM_RELEASE"
DEB_ARM_ASAN = "DEB_ARM_ASAN"
style_check_job = Job.Config( style_check_job = Job.Config(
name=JobNames.STYLE_CHECK, name=JobNames.STYLE_CHECK,
@ -51,7 +41,7 @@ fast_test_job = Job.Config(
build_jobs = Job.Config( build_jobs = Job.Config(
name=JobNames.BUILD, name=JobNames.BUILD,
runs_on=["...from params..."], runs_on=["...from params..."],
requires=[], requires=[JobNames.FAST_TEST],
command="python3 ./ci/jobs/build_clickhouse.py --build-type {PARAMETER}", command="python3 ./ci/jobs/build_clickhouse.py --build-type {PARAMETER}",
run_in_docker="clickhouse/fasttest", run_in_docker="clickhouse/fasttest",
timeout=3600 * 2, timeout=3600 * 2,
@ -73,26 +63,10 @@ build_jobs = Job.Config(
).parametrize( ).parametrize(
parameter=["amd_debug", "amd_release", "arm_release", "arm_asan"], parameter=["amd_debug", "amd_release", "arm_release", "arm_asan"],
provides=[ provides=[
[ [ArtifactNames.CH_AMD_DEBUG],
ArtifactNames.CH_AMD_DEBUG, [ArtifactNames.CH_AMD_RELEASE],
ArtifactNames.DEB_AMD_DEBUG, [ArtifactNames.CH_ARM_RELEASE],
ArtifactNames.CH_ODBC_B_AMD_DEBUG, [ArtifactNames.CH_ARM_ASAN],
],
[
ArtifactNames.CH_AMD_RELEASE,
ArtifactNames.DEB_AMD_RELEASE,
ArtifactNames.CH_ODBC_B_AMD_RELEASE,
],
[
ArtifactNames.CH_ARM_RELEASE,
ArtifactNames.DEB_ARM_RELEASE,
ArtifactNames.CH_ODBC_B_ARM_RELEASE,
],
[
ArtifactNames.CH_ARM_ASAN,
ArtifactNames.DEB_ARM_ASAN,
ArtifactNames.CH_ODBC_B_ARM_ASAN,
],
], ],
runs_on=[ runs_on=[
[RunnerLabels.BUILDER_AMD], [RunnerLabels.BUILDER_AMD],
@ -131,12 +105,12 @@ stateless_tests_jobs = Job.Config(
[RunnerLabels.FUNC_TESTER_ARM], [RunnerLabels.FUNC_TESTER_ARM],
], ],
requires=[ requires=[
[ArtifactNames.CH_AMD_DEBUG, ArtifactNames.CH_ODBC_B_AMD_DEBUG], [ArtifactNames.CH_AMD_DEBUG],
[ArtifactNames.CH_AMD_DEBUG, ArtifactNames.CH_ODBC_B_AMD_DEBUG], [ArtifactNames.CH_AMD_DEBUG],
[ArtifactNames.CH_AMD_RELEASE, ArtifactNames.CH_ODBC_B_AMD_RELEASE], [ArtifactNames.CH_AMD_RELEASE],
[ArtifactNames.CH_AMD_RELEASE, ArtifactNames.CH_ODBC_B_AMD_RELEASE], [ArtifactNames.CH_AMD_RELEASE],
[ArtifactNames.CH_ARM_ASAN, ArtifactNames.CH_ODBC_B_ARM_ASAN], [ArtifactNames.CH_ARM_ASAN],
[ArtifactNames.CH_ARM_ASAN, ArtifactNames.CH_ODBC_B_ARM_ASAN], [ArtifactNames.CH_ARM_ASAN],
], ],
) )
@ -154,7 +128,7 @@ stateful_tests_jobs = Job.Config(
), ),
).parametrize( ).parametrize(
parameter=[ parameter=[
"amd_release,parallel", "amd_debug,parallel",
], ],
runs_on=[ runs_on=[
[RunnerLabels.BUILDER_AMD], [RunnerLabels.BUILDER_AMD],
@ -164,29 +138,6 @@ stateful_tests_jobs = Job.Config(
], ],
) )
# TODO: refactor job to be aligned with praktika style (remove wrappers, run in docker)
stress_test_jobs = Job.Config(
name=JobNames.STRESS,
runs_on=[RunnerLabels.BUILDER_ARM],
command="python3 ./tests/ci/stress_check.py {PARAMETER}",
digest_config=Job.CacheDigestConfig(
include_paths=[
"./ci/jobs/functional_stateful_tests.py",
],
),
).parametrize(
parameter=[
"arm_release",
],
runs_on=[
[RunnerLabels.FUNC_TESTER_ARM],
],
requires=[
[ArtifactNames.DEB_ARM_RELEASE],
],
)
workflow = Workflow.Config( workflow = Workflow.Config(
name="PR", name="PR",
event=Workflow.Event.PULL_REQUEST, event=Workflow.Event.PULL_REQUEST,
@ -197,52 +148,27 @@ workflow = Workflow.Config(
*build_jobs, *build_jobs,
*stateless_tests_jobs, *stateless_tests_jobs,
*stateful_tests_jobs, *stateful_tests_jobs,
*stress_test_jobs,
], ],
artifacts=[ artifacts=[
*Artifact.Config( Artifact.Config(
name="...", name=ArtifactNames.CH_AMD_DEBUG,
type=Artifact.Type.S3, type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse", path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
).parametrize(
names=[
ArtifactNames.CH_AMD_DEBUG,
ArtifactNames.CH_AMD_RELEASE,
ArtifactNames.CH_ARM_RELEASE,
ArtifactNames.CH_ARM_ASAN,
]
),
*Artifact.Config(
name="...",
type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/build/programs/clickhouse-odbc-bridge",
).parametrize(
names=[
ArtifactNames.CH_ODBC_B_AMD_DEBUG,
ArtifactNames.CH_ODBC_B_AMD_RELEASE,
ArtifactNames.CH_ODBC_B_ARM_RELEASE,
ArtifactNames.CH_ODBC_B_ARM_ASAN,
]
), ),
Artifact.Config( Artifact.Config(
name=ArtifactNames.DEB_AMD_DEBUG, name=ArtifactNames.CH_AMD_RELEASE,
type=Artifact.Type.S3, type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/output/*.deb", path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
), ),
Artifact.Config( Artifact.Config(
name=ArtifactNames.DEB_AMD_RELEASE, name=ArtifactNames.CH_ARM_RELEASE,
type=Artifact.Type.S3, type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/output/*.deb", path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
), ),
Artifact.Config( Artifact.Config(
name=ArtifactNames.DEB_ARM_RELEASE, name=ArtifactNames.CH_ARM_ASAN,
type=Artifact.Type.S3, type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/output/*.deb", path=f"{Settings.TEMP_DIR}/build/programs/clickhouse",
),
Artifact.Config(
name=ArtifactNames.DEB_ARM_ASAN,
type=Artifact.Type.S3,
path=f"{Settings.TEMP_DIR}/output/*.deb",
), ),
], ],
dockers=DOCKERS, dockers=DOCKERS,

50
packages/build
View File

@ -5,14 +5,24 @@ set -e
# Avoid dependency on locale # Avoid dependency on locale
LC_ALL=C LC_ALL=C
# Normalize output directory
if [ -n "$OUTPUT_DIR" ]; then
OUTPUT_DIR=$(realpath -m "$OUTPUT_DIR")
fi
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
cd "$CUR_DIR"
ROOT_DIR=$(readlink -f "$(git rev-parse --show-cdup)")
PKG_ROOT='root' PKG_ROOT='root'
DEB_ARCH=${DEB_ARCH:-amd64} DEB_ARCH=${DEB_ARCH:-amd64}
OUTPUT_DIR=${OUTPUT_DIR:-$ROOT_DIR}
[ -d "${OUTPUT_DIR}" ] || mkdir -p "${OUTPUT_DIR}"
SANITIZER=${SANITIZER:-""} SANITIZER=${SANITIZER:-""}
SOURCE=${SOURCE:-$PKG_ROOT} SOURCE=${SOURCE:-$PKG_ROOT}
cd "$(dirname "${BASH_SOURCE[0]}")"
HELP="${0} [--test] [--rpm] [-h|--help] HELP="${0} [--test] [--rpm] [-h|--help]
--test - adds '+test' prefix to version --test - adds '+test' prefix to version
--apk - build APK packages --apk - build APK packages
@ -30,7 +40,12 @@ Used envs:
VERSION_STRING='${VERSION_STRING}' - the package version to overwrite VERSION_STRING='${VERSION_STRING}' - the package version to overwrite
" "
CLICKHOUSE_VERSION_STRING=${VERSION_STRING} if [ -z "${VERSION_STRING}" ]; then
# Get CLICKHOUSE_VERSION_STRING from the current git repo
eval "$("$ROOT_DIR/tests/ci/version_helper.py" -e)"
else
CLICKHOUSE_VERSION_STRING=${VERSION_STRING}
fi
export CLICKHOUSE_VERSION_STRING export CLICKHOUSE_VERSION_STRING
@ -129,32 +144,31 @@ CLICKHOUSE_VERSION_STRING+=$VERSION_POSTFIX
echo -e "\nCurrent version is $CLICKHOUSE_VERSION_STRING" echo -e "\nCurrent version is $CLICKHOUSE_VERSION_STRING"
for config in clickhouse*.yaml; do for config in clickhouse*.yaml; do
if [[ $BUILD_TYPE != 'release' ]] && [[ "$config" == "clickhouse-keeper-dbg.yaml" ]]; then
continue
fi
if [ -n "$MAKE_DEB" ] || [ -n "$MAKE_TGZ" ]; then if [ -n "$MAKE_DEB" ] || [ -n "$MAKE_TGZ" ]; then
echo "Building deb package for $config" echo "Building deb package for $config"
PKG_PATH=$(nfpm package --target "$OUTPUT_DIR" --config "$config" --packager deb | tee /dev/stderr | grep "created package:" | sed 's/.*created package: //')
# Preserve package path
exec 9>&1
PKG_PATH=$(nfpm package --target "$OUTPUT_DIR" --config "$config" --packager deb | tee /dev/fd/9)
PKG_PATH=${PKG_PATH##*created package: }
exec 9>&-
fi fi
if [ -n "$MAKE_APK" ]; then if [ -n "$MAKE_APK" ]; then
echo "Building apk package for $config" echo "Building apk package for $config"
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager apk nfpm package --target "$OUTPUT_DIR" --config "$config" --packager apk
fi fi
if [ -n "$MAKE_ARCHLINUX" ]; then if [ -n "$MAKE_ARCHLINUX" ]; then
echo "Building archlinux package for $config" echo "Building archlinux package for $config"
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager archlinux nfpm package --target "$OUTPUT_DIR" --config "$config" --packager archlinux
fi fi
if [ -n "$MAKE_RPM" ]; then if [ -n "$MAKE_RPM" ]; then
echo "Building rpm package for $config" echo "Building rpm package for $config"
nfpm package --target "$OUTPUT_DIR" --config "$config" --packager rpm nfpm package --target "$OUTPUT_DIR" --config "$config" --packager rpm
fi fi
if [ -n "$MAKE_TGZ" ]; then if [ -n "$MAKE_TGZ" ]; then
echo "Building tarball for $config" echo "Building tarball for $config"
deb2tgz "$PKG_PATH" deb2tgz "$PKG_PATH"
fi fi
done done

42
tests/ci/report.py
View File

@ -4,7 +4,6 @@ import datetime
import json import json
import logging import logging
import os import os
import sys
from ast import literal_eval from ast import literal_eval
from dataclasses import asdict, dataclass from dataclasses import asdict, dataclass
from html import escape from html import escape
@ -415,46 +414,6 @@ class JobReport:
dummy: bool = False dummy: bool = False
exit_code: int = -1 exit_code: int = -1
def to_praktika_result(self, job_name):
sys.path.append("./ci")
# ugly WA to exclude ci.py file form import
current_dir = os.path.dirname(os.path.abspath(__file__))
if current_dir in sys.path:
sys.path.remove(current_dir)
from praktika.result import ( # pylint: disable=import-error,import-outside-toplevel
Result,
)
if self.start_time:
dt = datetime.datetime.strptime(self.start_time, "%Y-%m-%d %H:%M:%S")
timestamp = dt.timestamp()
else:
timestamp = None
sub_results = []
for r in self.test_results:
sub_results.append(
Result(
name=r.name,
status=r.status,
info=r.raw_logs,
links=list(r.log_urls) if r.log_urls else [],
duration=r.time,
)
)
return Result(
name=job_name,
status=self.status,
start_time=timestamp,
duration=self.duration,
results=sub_results,
files=(
[str(f) for f in self.additional_files] if self.additional_files else []
),
)
@staticmethod @staticmethod
def get_start_time_from_current(): def get_start_time_from_current():
return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S") return datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
@ -515,7 +474,6 @@ class JobReport:
to_file = to_file or JOB_REPORT_FILE to_file = to_file or JOB_REPORT_FILE
with open(to_file, "w", encoding="utf-8") as json_file: with open(to_file, "w", encoding="utf-8") as json_file:
json.dump(asdict(self), json_file, default=path_converter, indent=2) json.dump(asdict(self), json_file, default=path_converter, indent=2)
return self
def read_test_results(results_path: Path, with_raw_logs: bool = True) -> TestResults: def read_test_results(results_path: Path, with_raw_logs: bool = True) -> TestResults:

16
tests/ci/stress_check.py
View File

@ -9,8 +9,6 @@ import sys
from pathlib import Path from pathlib import Path
from typing import List, Tuple from typing import List, Tuple
from praktika.utils import Shell # pylint: disable=import-error
from build_download_helper import download_all_deb_packages from build_download_helper import download_all_deb_packages
from clickhouse_helper import CiLogsCredentials from clickhouse_helper import CiLogsCredentials
from docker_images_helper import DockerImage, get_docker_image, pull_image from docker_images_helper import DockerImage, get_docker_image, pull_image
@ -156,19 +154,12 @@ def run_stress_test(upgrade_check: bool = False) -> None:
pr_info = PRInfo() pr_info = PRInfo()
docker_image = pull_image(get_docker_image("clickhouse/stateful-test")) docker_image = pull_image(get_docker_image("clickhouse/stress-test"))
packages_path = temp_path / "packages" packages_path = temp_path / "packages"
packages_path.mkdir(parents=True, exist_ok=True) packages_path.mkdir(parents=True, exist_ok=True)
if check_name in ("amd_release", "amd_debug", "arm_release"): download_all_deb_packages(check_name, reports_path, packages_path)
# this is praktika based CI
print("Copy input *.deb artifacts")
assert Shell.check(
f"cp /tmp/praktika/input/*.deb {packages_path}", verbose=True
)
else:
download_all_deb_packages(check_name, reports_path, packages_path)
server_log_path = temp_path / "server_log" server_log_path = temp_path / "server_log"
server_log_path.mkdir(parents=True, exist_ok=True) server_log_path.mkdir(parents=True, exist_ok=True)
@ -210,7 +201,6 @@ def run_stress_test(upgrade_check: bool = False) -> None:
result_path, server_log_path, run_log_path result_path, server_log_path, run_log_path
) )
Shell.check("pwd", verbose=True)
JobReport( JobReport(
description=description, description=description,
test_results=test_results, test_results=test_results,
@ -218,7 +208,7 @@ def run_stress_test(upgrade_check: bool = False) -> None:
start_time=stopwatch.start_time_str, start_time=stopwatch.start_time_str,
duration=stopwatch.duration_seconds, duration=stopwatch.duration_seconds,
additional_files=additional_logs, additional_files=additional_logs,
).dump().to_praktika_result(job_name=f"Stress tests ({check_name})").dump() ).dump()
if state == "failure": if state == "failure":
sys.exit(1) sys.exit(1)

2
tests/config/install.sh
View File

@ -21,7 +21,7 @@ while [[ "$#" -gt 0 ]]; do
--fast-test) FAST_TEST=1 ;; --fast-test) FAST_TEST=1 ;;
--s3-storage) EXPORT_S3_STORAGE_POLICIES=1 ;; --s3-storage) EXPORT_S3_STORAGE_POLICIES=1 ;;
--no-azure) NO_AZURE=1 ;; --no-azure) NO_AZURE=1 ;;
*) echo "Unknown option: $1" ;; *) echo "Unknown option: $1" ; exit 1 ;;
esac esac
shift shift
done done