Add instance_type column to CI Logs and the checks table

docker/test/base/setup_export_logs.sh

@@ -15,8 +15,8 @@ CLICKHOUSE_CI_LOGS_USER=${CLICKHOUSE_CI_LOGS_USER:-ci}
 # Pre-configured destination cluster, where to export the data
 CLICKHOUSE_CI_LOGS_CLUSTER=${CLICKHOUSE_CI_LOGS_CLUSTER:-system_logs_export}
 
-EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime, check_name LowCardinality(String), instance_type LowCardinality(String), "}
+EXTRA_COLUMNS=${EXTRA_COLUMNS:-"pull_request_number UInt32, commit_sha String, check_start_time DateTime, check_name LowCardinality(String), instance_type LowCardinality(String), instance_id String, "}
-EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"0 AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type"}
+EXTRA_COLUMNS_EXPRESSION=${EXTRA_COLUMNS_EXPRESSION:-"0 AS pull_request_number, '' AS commit_sha, now() AS check_start_time, '' AS check_name, '' AS instance_type, '' AS instance_id"}
 EXTRA_ORDER_BY_COLUMNS=${EXTRA_ORDER_BY_COLUMNS:-"check_name, "}
 
 function __set_connection_args
@@ -125,9 +125,9 @@ function setup_logs_replication
     echo 'Create %_log tables'
     clickhouse-client --query "SHOW TABLES FROM system LIKE '%\\_log'" | while read -r table
     do
-        # Calculate hash of its structure:
+        # Calculate hash of its structure. Note: 1 is the version of extra columns - increment it if extra columns are changed:
         hash=$(clickhouse-client --query "
-            SELECT sipHash64(groupArray((name, type)))
+            SELECT sipHash64(1, groupArray((name, type)))
             FROM (SELECT name, type FROM system.columns
                 WHERE database = 'system' AND table = '$table'
                 ORDER BY position)

tests/ci/build_check.py

@@ -34,6 +34,7 @@ from clickhouse_helper import (
     CiLogsCredentials,
     prepare_tests_results_for_clickhouse,
     get_instance_type,
+    get_instance_id,
 )
 from stopwatch import Stopwatch
 
@@ -364,6 +365,7 @@ def main():
     ci_logs_credentials = CiLogsCredentials(Path("/dev/null"))
     if ci_logs_credentials.host:
         instance_type = get_instance_type()
+        instance_id = get_instance_id()
         query = f"""INSERT INTO build_time_trace
 (
     pull_request_number,
@@ -371,6 +373,7 @@ def main():
     check_start_time,
     check_name,
     instance_type,
+    instance_id,
     file,
     library,
     time,
@@ -386,7 +389,7 @@ def main():
     avgMs,
     args_name
 )
-SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', *
+SELECT {pr_info.number}, '{pr_info.sha}', '{stopwatch.start_time_str}', '{build_name}', '{instance_type}', '{instance_id}', *
 FROM input('
     file String,
     library String,

tests/ci/clickhouse_helper.py

@@ -168,9 +168,8 @@ class ClickHouseHelper:
         return result
 
 
-# Obtain the machine type from IMDS:
+def _query_imds(path):
-def get_instance_type():
+    url = f"http://169.254.169.254/{path}"
-    url = "http://169.254.169.254/latest/meta-data/instance-type"
     for i in range(5):
         try:
             response = requests.get(url, timeout=1)
@@ -184,6 +183,14 @@ def get_instance_type():
             continue
     return ""
 
+# Obtain the machine type from IMDS:
+def get_instance_type():
+    return _query_imds("latest/meta-data/instance-type")
+
+# Obtain the instance id from IMDS:
+def get_instance_id():
+    return _query_imds("latest/meta-data/instance-id")
+
 
 def prepare_tests_results_for_clickhouse(
     pr_info: PRInfo,
@@ -222,6 +229,7 @@ def prepare_tests_results_for_clickhouse(
         head_repo=head_repo,
         task_url=pr_info.task_url,
         instance_type=get_instance_type(),
+        instance_id=get_instance_id(),
     )
 
     # Always publish a total record for all checks. For checks with individual
@@ -288,7 +296,7 @@ class CiLogsCredentials:
         extra_columns = (
             f"{pr_info.number} AS pull_request_number, '{pr_info.sha}' AS commit_sha, "
             f"toDateTime('{check_start_time}', 'UTC') AS check_start_time, '{check_name}' AS check_name, "
-            f"'{get_instance_type()}' AS instance_type"
+            f"'{get_instance_type()}' AS instance_type, '{get_instance_id()}' AS instance_id, "
         )
         return (
             f'-e EXTRA_COLUMNS_EXPRESSION="{extra_columns}" '

tests/ci/performance_comparison_check.py

@@ -20,7 +20,7 @@ from get_robot_token import get_best_robot_token, get_parameter_from_ssm
 from pr_info import PRInfo
 from s3_helper import S3Helper
 from tee_popen import TeePopen
-from clickhouse_helper import get_instance_type
+from clickhouse_helper import get_instance_type, get_instance_id
 from stopwatch import Stopwatch
 
 IMAGE_NAME = "clickhouse/performance-comparison"
@@ -38,11 +38,13 @@ def get_run_command(
     image,
 ):
     instance_type = get_instance_type()
+    instance_id = get_instance_id()
 
     envs = [
         f"-e CHECK_START_TIME='{check_start_time}'",
         f"-e CHECK_NAME='{check_name}'",
         f"-e INSTANCE_TYPE='{instance_type}'",
+        f"-e INSTANCE_ID='{instance_id}'",
         f"-e PR_TO_TEST={pr_to_test}",
         f"-e SHA_TO_TEST={sha_to_test}",
     ]