Export logs from CI in performance (part 2)

docker/test/base/setup_export_logs.sh

@@ -7,6 +7,7 @@
 
 # Pre-configured destination cluster, where to export the data
 CLUSTER=${CLUSTER:=system_logs_export}
+LOCAL_PARAMETERS=$1
 
 EXTRA_COLUMNS=${EXTRA_COLUMNS:="pull_request_number UInt32, commit_sha String, check_start_time DateTime, check_name LowCardinality(String), instance_type LowCardinality(String), "}
 EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:="0 AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type"}
@@ -15,13 +16,13 @@ EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:="check_name, "}
 CONNECTION_PARAMETERS=${CONNECTION_PARAMETERS:=""}
 
 # Create all configured system logs:
-clickhouse-client --query "SYSTEM FLUSH LOGS"
+clickhouse-client $LOCAL_PARAMETERS --query "SYSTEM FLUSH LOGS"
 
 # For each system log table:
-clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
+clickhouse-client $LOCAL_PARAMETERS --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
 do
     # Calculate hash of its structure:
-    hash=$(clickhouse-client --query "
+    hash=$(clickhouse-client $LOCAL_PARAMETERS --query "
         SELECT sipHash64(groupArray((name, type)))
         FROM (SELECT name, type FROM system.columns
             WHERE database = 'system' AND table = '$table'
@@ -29,7 +30,7 @@ do
         ")
 
     # Create the destination table with adapted name and structure:
-    statement=$(clickhouse-client --format TSVRaw --query "SHOW CREATE TABLE system.${table}" | sed -r -e '
+    statement=$(clickhouse-client $LOCAL_PARAMETERS --format TSVRaw --query "SHOW CREATE TABLE system.${table}" | sed -r -e '
         s/^\($/('"$EXTRA_COLUMNS"'/;
         s/ORDER BY \(/ORDER BY ('"$EXTRA_ORDER_BY_COLUMNS"'/;
         s/^CREATE TABLE system\.\w+_log$/CREATE TABLE IF NOT EXISTS '"$table"'_'"$hash"'/;
@@ -43,7 +44,7 @@ do
     echo "Creating table system.${table}_sender" >&2
 
     # Create Distributed table and materialized view to watch on the original table:
-    clickhouse-client --query "
+    clickhouse-client $LOCAL_PARAMETERS --query "
         CREATE TABLE system.${table}_sender
         ENGINE = Distributed(${CLUSTER}, default, ${table}_${hash})
         EMPTY AS
@@ -53,7 +54,7 @@ do
 
     echo "Creating materialized view system.${table}_watcher" >&2
 
-    clickhouse-client --query "
+    clickhouse-client $LOCAL_PARAMETERS --query "
         CREATE MATERIALIZED VIEW system.${table}_watcher TO system.${table}_sender AS
         SELECT ${EXTRA_COLUMNS_EXPRESSION}, *
         FROM system.${table}

docker/test/performance-comparison/compare.sh

@@ -63,6 +63,22 @@ function left_or_right()
 
 function configure
 {
+    # Setup a cluster for logs export to ClickHouse Cloud
+    # Note: these variables are provided to the Docker run command by the Python script in tests/ci
+    if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+    then
+        echo "
+remote_servers:
+    system_logs_export:
+        shard:
+            replica:
+                secure: 1
+                user: ci
+                host: '${CLICKHOUSE_CI_LOGS_HOST}'
+                password: '${CLICKHOUSE_CI_LOGS_PASSWORD}'
+" > right/config/config.d/system_logs_export.yaml
+    fi
+
     # Use the new config for both servers, so that we can change it in a PR.
     rm right/config/config.d/text_log.xml ||:
     cp -rv right/config left ||:
@@ -71,8 +87,6 @@ function configure
     while pkill -f clickhouse-serv ; do echo . ; sleep 1 ; done
     echo all killed
 
-
-
     set -m # Spawn temporary in its own process groups
 
     local setup_left_server_opts=(
@@ -92,7 +106,22 @@ function configure
     set +m
 
     wait_for_server $LEFT_SERVER_PORT $left_pid
-    echo Server for setup started
+    echo "Server for setup started"
+
+    # Initialize export of system logs to ClickHouse Cloud
+    # Note: it is set up for the "left" server, and its database is then cloned to the "right" server.
+    if [ -n "${CLICKHOUSE_CI_LOGS_HOST}" ]
+    then
+        export EXTRA_COLUMNS_EXPRESSION="$PR_TO_TEST AS pull_request_number, '$SHA_TO_TEST' AS commit_sha, '$CHECK_START_TIME' AS check_start_time, '$CHECK_NAME' AS check_name, '$INSTANCE_TYPE' AS instance_type"
+        export CONNECTION_PARAMETERS="--secure --user ci --host ${CLICKHOUSE_CI_LOGS_HOST} --password ${CLICKHOUSE_CI_LOGS_PASSWORD}"
+
+        ./setup_export_logs.sh "--port $LEFT_SERVER_PORT"
+
+        # Unset variables after use
+        export CONNECTION_PARAMETERS=''
+        export CLICKHOUSE_CI_LOGS_HOST=''
+        export CLICKHOUSE_CI_LOGS_PASSWORD=''
+    fi
 
     clickhouse-client --port $LEFT_SERVER_PORT --query "create database test" ||:
     clickhouse-client --port $LEFT_SERVER_PORT --query "rename table datasets.hits_v1 to test.hits" ||: