mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-11-21 15:12:02 +00:00
Revert "Upload build time-trace data to CI database"
This commit is contained in:
Alexander Gololobov
GitHub
parent
48ed54e822
commit
63183c56f1
3
.github/workflows/backport_branches.yml
vendored
3
.github/workflows/backport_branches.yml
vendored
@ -3,9 +3,6 @@ name: BackportPR
|
||||
env:
|
||||
# Force the stdout and stderr streams to be unbuffered
|
||||
PYTHONUNBUFFERED: 1
|
||||
# Export system tables to ClickHouse Cloud
|
||||
CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
|
||||
CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
|
||||
|
||||
on: # yamllint disable-line rule:truthy
|
||||
push:
|
||||
|
||||
3
.github/workflows/master.yml
vendored
3
.github/workflows/master.yml
vendored
@ -3,9 +3,6 @@ name: MasterCI
|
||||
env:
|
||||
# Force the stdout and stderr streams to be unbuffered
|
||||
PYTHONUNBUFFERED: 1
|
||||
# Export system tables to ClickHouse Cloud
|
||||
CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
|
||||
CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
|
||||
|
||||
on: # yamllint disable-line rule:truthy
|
||||
push:
|
||||
|
||||
3
.github/workflows/pull_request.yml
vendored
3
.github/workflows/pull_request.yml
vendored
@ -3,9 +3,6 @@ name: PullRequestCI
|
||||
env:
|
||||
# Force the stdout and stderr streams to be unbuffered
|
||||
PYTHONUNBUFFERED: 1
|
||||
# Export system tables to ClickHouse Cloud
|
||||
CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
|
||||
CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
|
||||
|
||||
on: # yamllint disable-line rule:truthy
|
||||
pull_request:
|
||||
|
||||
3
.github/workflows/release_branches.yml
vendored
3
.github/workflows/release_branches.yml
vendored
@ -3,9 +3,6 @@ name: ReleaseBranchCI
|
||||
env:
|
||||
# Force the stdout and stderr streams to be unbuffered
|
||||
PYTHONUNBUFFERED: 1
|
||||
# Export system tables to ClickHouse Cloud
|
||||
CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
|
||||
CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
|
||||
|
||||
on: # yamllint disable-line rule:truthy
|
||||
push:
|
||||
|
||||
@ -101,7 +101,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
|
||||
python3-boto3 \
|
||||
yasm \
|
||||
zstd \
|
||||
jq \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists
|
||||
|
||||
|
||||
@ -59,7 +59,7 @@ if [ "$BUILD_MUSL_KEEPER" == "1" ]
|
||||
then
|
||||
# build keeper with musl separately
|
||||
# and without rust bindings
|
||||
cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
|
||||
cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
|
||||
# shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
|
||||
ninja $NINJA_FLAGS clickhouse-keeper
|
||||
|
||||
@ -74,10 +74,10 @@ then
|
||||
rm -f CMakeCache.txt
|
||||
|
||||
# Build the rest of binaries
|
||||
cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
|
||||
cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
|
||||
else
|
||||
# Build everything
|
||||
cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
|
||||
cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
|
||||
fi
|
||||
|
||||
# No quotes because I want it to expand to nothing if empty.
|
||||
@ -181,11 +181,4 @@ then
|
||||
tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
|
||||
fi
|
||||
|
||||
# Prepare profile info (time-trace)
|
||||
mkdir -p profile-tmp
|
||||
../utils/prepare-time-trace/prepare-time-trace.sh . profile-tmp
|
||||
find profile-tmp -type f -print0 | xargs -0 cat > /profile/profile.json
|
||||
|
||||
wc -c /profile/profile.json
|
||||
|
||||
ls -l /output
|
||||
|
||||
@ -78,14 +78,11 @@ def run_docker_image_with_env(
|
||||
image_name: str,
|
||||
as_root: bool,
|
||||
output_dir: Path,
|
||||
profile_dir: Path,
|
||||
env_variables: List[str],
|
||||
ch_root: Path,
|
||||
ccache_dir: Optional[Path],
|
||||
):
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
profile_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
env_part = " -e ".join(env_variables)
|
||||
if env_part:
|
||||
env_part = " -e " + env_part
|
||||
@ -106,7 +103,7 @@ def run_docker_image_with_env(
|
||||
|
||||
cmd = (
|
||||
f"docker run --network=host --user={user} --rm {ccache_mount}"
|
||||
f"--volume={output_dir}:/output --volume={ch_root}:/build --volume={profile_dir}:/profile {env_part} "
|
||||
f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
|
||||
f"{interactive} {image_name}"
|
||||
)
|
||||
|
||||
@ -364,7 +361,6 @@ def parse_args() -> argparse.Namespace:
|
||||
help="ClickHouse git repository",
|
||||
)
|
||||
parser.add_argument("--output-dir", type=dir_name, required=True)
|
||||
parser.add_argument("--profile-dir", type=dir_name, required=True)
|
||||
parser.add_argument("--debug-build", action="store_true")
|
||||
|
||||
parser.add_argument(
|
||||
@ -492,7 +488,6 @@ def main():
|
||||
image_with_version,
|
||||
args.as_root,
|
||||
args.output_dir,
|
||||
args.profile_dir,
|
||||
env_prepared,
|
||||
ch_root,
|
||||
args.ccache_dir,
|
||||
|
||||
@ -396,9 +396,9 @@ std::unique_ptr<ReadBuffer> createReadBuffer(
|
||||
throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP,
|
||||
"Cannot compile regex from glob ({}): {}", current_path, matcher->error());
|
||||
|
||||
return reader->readFile([my_matcher = std::move(matcher)](const std::string & path)
|
||||
return reader->readFile([matcher = std::move(matcher)](const std::string & path)
|
||||
{
|
||||
return re2::RE2::FullMatch(path, *my_matcher);
|
||||
return re2::RE2::FullMatch(path, *matcher);
|
||||
});
|
||||
}
|
||||
else
|
||||
|
||||
@ -1,14 +1,12 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
from typing import List, Tuple
|
||||
import subprocess
|
||||
import logging
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
import requests # type: ignore
|
||||
from typing import List, Tuple
|
||||
|
||||
from ci_config import CI_CONFIG, BuildConfig
|
||||
from docker_pull_helper import get_image_with_version
|
||||
@ -32,7 +30,6 @@ from version_helper import (
|
||||
from clickhouse_helper import (
|
||||
ClickHouseHelper,
|
||||
prepare_tests_results_for_clickhouse,
|
||||
get_instance_type,
|
||||
)
|
||||
from stopwatch import Stopwatch
|
||||
|
||||
@ -54,7 +51,6 @@ def get_packager_cmd(
|
||||
build_config: BuildConfig,
|
||||
packager_path: str,
|
||||
output_path: str,
|
||||
profile_path: str,
|
||||
build_version: str,
|
||||
image_version: str,
|
||||
official: bool,
|
||||
@ -63,8 +59,8 @@ def get_packager_cmd(
|
||||
comp = build_config.compiler
|
||||
cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1"
|
||||
cmd = (
|
||||
f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} --profile-dir={profile_path}"
|
||||
f" --package-type={package_type} --compiler={comp}"
|
||||
f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} "
|
||||
f"--package-type={package_type} --compiler={comp}"
|
||||
)
|
||||
|
||||
if build_config.debug_build:
|
||||
@ -290,15 +286,10 @@ def main():
|
||||
if not os.path.exists(build_output_path):
|
||||
os.makedirs(build_output_path)
|
||||
|
||||
build_profile_path = os.path.join(TEMP_PATH, f"{build_name}_profile")
|
||||
if not os.path.exists(build_profile_path):
|
||||
os.makedirs(build_profile_path)
|
||||
|
||||
packager_cmd = get_packager_cmd(
|
||||
build_config,
|
||||
os.path.join(REPO_COPY, "docker/packager"),
|
||||
build_output_path,
|
||||
build_profile_path,
|
||||
version.string,
|
||||
image_version,
|
||||
official_flag,
|
||||
@ -369,69 +360,6 @@ def main():
|
||||
|
||||
upload_master_static_binaries(pr_info, build_config, s3_helper, build_output_path)
|
||||
|
||||
# Upload profile data
|
||||
|
||||
instance_type = get_instance_type()
|
||||
query = urllib.parse.quote(
|
||||
f"""
|
||||
INSERT INTO build_time_trace
|
||||
(
|
||||
pull_request_number,
|
||||
commit_sha,
|
||||
check_start_time,
|
||||
check_name,
|
||||
instance_type,
|
||||
file,
|
||||
library,
|
||||
time,
|
||||
pid,
|
||||
tid,
|
||||
ph,
|
||||
ts,
|
||||
dur,
|
||||
cat,
|
||||
name,
|
||||
detail,
|
||||
count,
|
||||
avgMs,
|
||||
args_name
|
||||
)
|
||||
SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', *
|
||||
FROM input('
|
||||
file String,
|
||||
library String,
|
||||
time DateTime64(6),
|
||||
pid UInt32,
|
||||
tid UInt32,
|
||||
ph String,
|
||||
ts UInt64,
|
||||
dur UInt64,
|
||||
cat String,
|
||||
name String,
|
||||
detail String,
|
||||
count UInt64,
|
||||
avgMs UInt64,
|
||||
args_name String')
|
||||
FORMAT JSONCompactEachRow
|
||||
"""
|
||||
)
|
||||
clickhouse_ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST")
|
||||
maybe_clickhouse_ci_logs_password: str = (
|
||||
os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD") or ""
|
||||
)
|
||||
url = f"https://{clickhouse_ci_logs_host}/?query={query}"
|
||||
file_path = os.path.join(build_profile_path, "profile.json")
|
||||
file_size = os.path.getsize(file_path)
|
||||
|
||||
print(
|
||||
f"::notice ::Log Uploading profile data, path: {file_path}, size: {file_size}, query: {query}"
|
||||
)
|
||||
|
||||
with open(file_path, "rb") as file:
|
||||
requests.post(url, data=file, auth=("ci", maybe_clickhouse_ci_logs_password))
|
||||
|
||||
# Upload statistics to CI database
|
||||
|
||||
ch_helper = ClickHouseHelper()
|
||||
prepared_events = prepare_tests_results_for_clickhouse(
|
||||
pr_info,
|
||||
|
||||
@ -35,6 +35,7 @@ ENGINE = MergeTree ORDER BY (date, file, name, args_name);
|
||||
|
||||
INPUT_DIR=$1
|
||||
OUTPUT_DIR=$2
|
||||
EXTRA_COLUMN_VALUES=$3
|
||||
|
||||
find "$INPUT_DIR" -name '*.json' | grep -P '\.(c|cpp|cc|cxx)\.json$' | xargs -P $(nproc) -I{} bash -c "
|
||||
|
||||
@ -42,7 +43,7 @@ find "$INPUT_DIR" -name '*.json' | grep -P '\.(c|cpp|cc|cxx)\.json$' | xargs -P
|
||||
LIBRARY_NAME=\$(echo '{}' | sed -r -e 's!^.*/CMakeFiles/([^/]+)\.dir/.*\$!\1!')
|
||||
START_TIME=\$(jq '.beginningOfTime' '{}')
|
||||
|
||||
jq -c '.traceEvents[] | [\"'\"\$ORIGINAL_FILENAME\"'\", \"'\"\$LIBRARY_NAME\"'\", '\$START_TIME', .pid, .tid, .ph, .ts, .dur, .cat, .name, .args.detail, .args.count, .args[\"avg ms\"], .args.name]' '{}' > \"${OUTPUT_DIR}/\$\$\"
|
||||
jq -c '.traceEvents[] | [${EXTRA_COLUMN_VALUES} \"'\"\$ORIGINAL_FILENAME\"'\", \"'\"\$LIBRARY_NAME\"'\", '\$START_TIME', .pid, .tid, .ph, .ts, .dur, .cat, .name, .args.detail, .args.count, .args[\"avg ms\"], .args.name]' '{}' > \"${OUTPUT_DIR}/\$\$\"
|
||||
"
|
||||
|
||||
# Now you can upload it as follows:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user