Revert "Upload build time-trace data to CI database"

.github/workflows/backport_branches.yml

@@ -3,9 +3,6 @@ name: BackportPR
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
-  # Export system tables to ClickHouse Cloud
-  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
-  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on: # yamllint disable-line rule:truthy
   push:

.github/workflows/master.yml

@@ -3,9 +3,6 @@ name: MasterCI
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
-  # Export system tables to ClickHouse Cloud
-  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
-  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on: # yamllint disable-line rule:truthy
   push:

.github/workflows/pull_request.yml

@@ -3,9 +3,6 @@ name: PullRequestCI
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
-  # Export system tables to ClickHouse Cloud
-  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
-  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on:  # yamllint disable-line rule:truthy
   pull_request:

.github/workflows/release_branches.yml

@@ -3,9 +3,6 @@ name: ReleaseBranchCI
 env:
   # Force the stdout and stderr streams to be unbuffered
   PYTHONUNBUFFERED: 1
-  # Export system tables to ClickHouse Cloud
-  CLICKHOUSE_CI_LOGS_HOST: ${{ secrets.CLICKHOUSE_CI_LOGS_HOST }}
-  CLICKHOUSE_CI_LOGS_PASSWORD: ${{ secrets.CLICKHOUSE_CI_LOGS_PASSWORD }}
 
 on: # yamllint disable-line rule:truthy
   push:

docker/packager/binary/Dockerfile

@@ -101,7 +101,6 @@ RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \
         python3-boto3 \
         yasm \
         zstd \
-        jq \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists
 

docker/packager/binary/build.sh

@@ -59,7 +59,7 @@ if [ "$BUILD_MUSL_KEEPER" == "1" ]
 then
     # build keeper with musl separately
     # and without rust bindings
-    cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DENABLE_RUST=OFF -DBUILD_STANDALONE_KEEPER=1 -DENABLE_CLICKHOUSE_KEEPER=1 -DCMAKE_VERBOSE_MAKEFILE=1 -DUSE_MUSL=1 -LA -DCMAKE_TOOLCHAIN_FILE=/build/cmake/linux/toolchain-x86_64-musl.cmake "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
     # shellcheck disable=SC2086 # No quotes because I want it to expand to nothing if empty.
     ninja $NINJA_FLAGS clickhouse-keeper
 
@@ -74,10 +74,10 @@ then
     rm -f CMakeCache.txt
 
     # Build the rest of binaries
-    cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DBUILD_STANDALONE_KEEPER=0 -DCREATE_KEEPER_SYMLINK=0 -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
 else
     # Build everything
-    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 -DENABLE_BUILD_PROFILING=1 "${CMAKE_FLAGS[@]}" ..
+    cmake --debug-trycompile -DCMAKE_VERBOSE_MAKEFILE=1 -LA "-DCMAKE_BUILD_TYPE=$BUILD_TYPE" "-DSANITIZE=$SANITIZER" -DENABLE_CHECK_HEAVY_BUILDS=1 "${CMAKE_FLAGS[@]}" ..
 fi
 
 # No quotes because I want it to expand to nothing if empty.
@@ -181,11 +181,4 @@ then
     tar -cv -I pixz -f /output/ccache.log.txz "$CCACHE_LOGFILE"
 fi
 
-# Prepare profile info (time-trace)
-mkdir -p profile-tmp
-../utils/prepare-time-trace/prepare-time-trace.sh . profile-tmp
-find profile-tmp -type f -print0 | xargs -0 cat > /profile/profile.json
-
-wc -c /profile/profile.json
-
 ls -l /output

docker/packager/packager

@@ -78,14 +78,11 @@ def run_docker_image_with_env(
     image_name: str,
     as_root: bool,
     output_dir: Path,
-    profile_dir: Path,
     env_variables: List[str],
     ch_root: Path,
     ccache_dir: Optional[Path],
 ):
     output_dir.mkdir(parents=True, exist_ok=True)
-    profile_dir.mkdir(parents=True, exist_ok=True)
-
     env_part = " -e ".join(env_variables)
     if env_part:
         env_part = " -e " + env_part
@@ -106,7 +103,7 @@ def run_docker_image_with_env(
 
     cmd = (
         f"docker run --network=host --user={user} --rm {ccache_mount}"
-        f"--volume={output_dir}:/output --volume={ch_root}:/build --volume={profile_dir}:/profile {env_part} "
+        f"--volume={output_dir}:/output --volume={ch_root}:/build {env_part} "
         f"{interactive} {image_name}"
     )
 
@@ -364,7 +361,6 @@ def parse_args() -> argparse.Namespace:
         help="ClickHouse git repository",
     )
     parser.add_argument("--output-dir", type=dir_name, required=True)
-    parser.add_argument("--profile-dir", type=dir_name, required=True)
     parser.add_argument("--debug-build", action="store_true")
 
     parser.add_argument(
@@ -492,7 +488,6 @@ def main():
         image_with_version,
         args.as_root,
         args.output_dir,
-        args.profile_dir,
         env_prepared,
         ch_root,
         args.ccache_dir,

src/Storages/StorageFile.cpp

@@ -396,9 +396,9 @@ std::unique_ptr<ReadBuffer> createReadBuffer(
                 throw Exception(ErrorCodes::CANNOT_COMPILE_REGEXP,
                     "Cannot compile regex from glob ({}): {}", current_path, matcher->error());
 
-            return reader->readFile([my_matcher = std::move(matcher)](const std::string & path)
+            return reader->readFile([matcher = std::move(matcher)](const std::string & path)
             {
-                return re2::RE2::FullMatch(path, *my_matcher);
+                return re2::RE2::FullMatch(path, *matcher);
             });
         }
         else

tests/ci/build_check.py

@@ -1,14 +1,12 @@
 #!/usr/bin/env python3
 
-from typing import List, Tuple
 import subprocess
 import logging
 import json
 import os
 import sys
 import time
-import urllib.parse
+from typing import List, Tuple
-import requests  # type: ignore
 
 from ci_config import CI_CONFIG, BuildConfig
 from docker_pull_helper import get_image_with_version
@@ -32,7 +30,6 @@ from version_helper import (
 from clickhouse_helper import (
     ClickHouseHelper,
     prepare_tests_results_for_clickhouse,
-    get_instance_type,
 )
 from stopwatch import Stopwatch
 
@@ -54,7 +51,6 @@ def get_packager_cmd(
     build_config: BuildConfig,
     packager_path: str,
     output_path: str,
-    profile_path: str,
     build_version: str,
     image_version: str,
     official: bool,
@@ -63,7 +59,7 @@ def get_packager_cmd(
     comp = build_config.compiler
     cmake_flags = "-DENABLE_CLICKHOUSE_SELF_EXTRACTING=1"
     cmd = (
-        f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} --profile-dir={profile_path}"
+        f"cd {packager_path} && CMAKE_FLAGS='{cmake_flags}' ./packager --output-dir={output_path} "
         f"--package-type={package_type} --compiler={comp}"
     )
 
@@ -290,15 +286,10 @@ def main():
     if not os.path.exists(build_output_path):
         os.makedirs(build_output_path)
 
-    build_profile_path = os.path.join(TEMP_PATH, f"{build_name}_profile")
-    if not os.path.exists(build_profile_path):
-        os.makedirs(build_profile_path)
-
     packager_cmd = get_packager_cmd(
         build_config,
         os.path.join(REPO_COPY, "docker/packager"),
         build_output_path,
-        build_profile_path,
         version.string,
         image_version,
         official_flag,
@@ -369,69 +360,6 @@ def main():
 
     upload_master_static_binaries(pr_info, build_config, s3_helper, build_output_path)
 
-    # Upload profile data
-
-    instance_type = get_instance_type()
-    query = urllib.parse.quote(
-        f"""
-        INSERT INTO build_time_trace
-        (
-            pull_request_number,
-            commit_sha,
-            check_start_time,
-            check_name,
-            instance_type,
-            file,
-            library,
-            time,
-            pid,
-            tid,
-            ph,
-            ts,
-            dur,
-            cat,
-            name,
-            detail,
-            count,
-            avgMs,
-            args_name
-        )
-        SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', *
-        FROM input('
-            file String,
-            library String,
-            time DateTime64(6),
-            pid UInt32,
-            tid UInt32,
-            ph String,
-            ts UInt64,
-            dur UInt64,
-            cat String,
-            name String,
-            detail String,
-            count UInt64,
-            avgMs UInt64,
-            args_name String')
-        FORMAT JSONCompactEachRow
-    """
-    )
-    clickhouse_ci_logs_host = os.getenv("CLICKHOUSE_CI_LOGS_HOST")
-    maybe_clickhouse_ci_logs_password: str = (
-        os.getenv("CLICKHOUSE_CI_LOGS_PASSWORD") or ""
-    )
-    url = f"https://{clickhouse_ci_logs_host}/?query={query}"
-    file_path = os.path.join(build_profile_path, "profile.json")
-    file_size = os.path.getsize(file_path)
-
-    print(
-        f"::notice ::Log Uploading profile data, path: {file_path}, size: {file_size}, query: {query}"
-    )
-
-    with open(file_path, "rb") as file:
-        requests.post(url, data=file, auth=("ci", maybe_clickhouse_ci_logs_password))
-
-    # Upload statistics to CI database
-
     ch_helper = ClickHouseHelper()
     prepared_events = prepare_tests_results_for_clickhouse(
         pr_info,

utils/prepare-time-trace/prepare-time-trace.sh

@@ -35,6 +35,7 @@ ENGINE = MergeTree ORDER BY (date, file, name, args_name);
 
 INPUT_DIR=$1
 OUTPUT_DIR=$2
+EXTRA_COLUMN_VALUES=$3
 
 find "$INPUT_DIR" -name '*.json' | grep -P '\.(c|cpp|cc|cxx)\.json$' | xargs -P $(nproc) -I{} bash -c "
 
@@ -42,7 +43,7 @@ find "$INPUT_DIR" -name '*.json' | grep -P '\.(c|cpp|cc|cxx)\.json$' | xargs -P
     LIBRARY_NAME=\$(echo '{}' | sed -r -e 's!^.*/CMakeFiles/([^/]+)\.dir/.*\$!\1!')
     START_TIME=\$(jq '.beginningOfTime' '{}')
 
-    jq -c '.traceEvents[] | [\"'\"\$ORIGINAL_FILENAME\"'\", \"'\"\$LIBRARY_NAME\"'\", '\$START_TIME', .pid, .tid, .ph, .ts, .dur, .cat, .name, .args.detail, .args.count, .args[\"avg ms\"], .args.name]' '{}' > \"${OUTPUT_DIR}/\$\$\"
+    jq -c '.traceEvents[] | [${EXTRA_COLUMN_VALUES} \"'\"\$ORIGINAL_FILENAME\"'\", \"'\"\$LIBRARY_NAME\"'\", '\$START_TIME', .pid, .tid, .ph, .ts, .dur, .cat, .name, .args.detail, .args.count, .args[\"avg ms\"], .args.name]' '{}' > \"${OUTPUT_DIR}/\$\$\"
 "
 
 # Now you can upload it as follows: