Merge pull request #67998 from ClickHouse/minio-audit-logs

Collect minio audit logs in stateless tests

docker/test/stateless/run.sh

@@ -54,8 +54,6 @@ source /utils.lib
 /usr/share/clickhouse-test/config/install.sh
 
 ./setup_minio.sh stateless
-./mc admin trace clickminio > /test_output/minio.log &
-MC_ADMIN_PID=$!
 
 ./setup_hdfs_minicluster.sh
 
@@ -176,6 +174,55 @@ done
 setup_logs_replication
 attach_gdb_to_clickhouse
 
+# create tables for minio log webhooks
+clickhouse-client --query "CREATE TABLE minio_audit_logs
+(
+    log String,
+    event_time DateTime64(9) MATERIALIZED parseDateTime64BestEffortOrZero(trim(BOTH '\"' FROM JSONExtractRaw(log, 'time')), 9, 'UTC')
+)
+ENGINE = MergeTree
+ORDER BY tuple()"
+
+clickhouse-client --query "CREATE TABLE minio_server_logs
+(
+    log String,
+    event_time DateTime64(9) MATERIALIZED parseDateTime64BestEffortOrZero(trim(BOTH '\"' FROM JSONExtractRaw(log, 'time')), 9, 'UTC')
+)
+ENGINE = MergeTree
+ORDER BY tuple()"
+
+# create minio log webhooks for both audit and server logs
+# use async inserts to avoid creating too many parts
+./mc admin config set clickminio logger_webhook:ch_server_webhook endpoint="http://localhost:8123/?async_insert=1&wait_for_async_insert=0&async_insert_busy_timeout_min_ms=5000&async_insert_busy_timeout_max_ms=5000&async_insert_max_query_number=1000&async_insert_max_data_size=10485760&query=INSERT%20INTO%20minio_server_logs%20FORMAT%20LineAsString" queue_size=1000000 batch_size=500
+./mc admin config set clickminio audit_webhook:ch_audit_webhook endpoint="http://localhost:8123/?async_insert=1&wait_for_async_insert=0&async_insert_busy_timeout_min_ms=5000&async_insert_busy_timeout_max_ms=5000&async_insert_max_query_number=1000&async_insert_max_data_size=10485760&query=INSERT%20INTO%20minio_audit_logs%20FORMAT%20LineAsString" queue_size=1000000 batch_size=500
+
+max_retries=100
+retry=1
+while [ $retry -le $max_retries ]; do
+    echo "clickminio restart attempt $retry:"
+
+    output=$(./mc admin service restart clickminio --wait --json 2>&1 | jq -r .status)
+    echo "Output of restart status: $output"
+
+    expected_output="success
+success"
+    if [ "$output" = "$expected_output" ]; then
+        echo "Restarted clickminio successfully."
+        break
+    fi
+
+    sleep 1
+
+    retry=$((retry + 1))
+done
+
+if [ $retry -gt $max_retries ]; then
+    echo "Failed to restart clickminio after $max_retries attempts."
+fi
+
+./mc admin trace clickminio > /test_output/minio.log &
+MC_ADMIN_PID=$!
+
 function fn_exists() {
     declare -F "$1" > /dev/null;
 }
@@ -339,6 +386,14 @@ do
     fi
 done
 
+
+# collect minio audit and server logs
+# wait for minio to flush its batch if it has any
+sleep 1
+clickhouse-client -q "SYSTEM FLUSH ASYNC INSERT QUEUE"
+clickhouse-client -q "SELECT log FROM minio_audit_logs ORDER BY event_time INTO OUTFILE '/test_output/minio_audit_logs.jsonl.zst' FORMAT JSONEachRow"
+clickhouse-client -q "SELECT log FROM minio_server_logs ORDER BY event_time INTO OUTFILE '/test_output/minio_server_logs.jsonl.zst' FORMAT JSONEachRow"
+
 # Stop server so we can safely read data with clickhouse-local.
 # Why do we read data with clickhouse-local?
 # Because it's the simplest way to read it when server has crashed.

tests/queries/0_stateless/02726_async_insert_flush_queue.sql

@@ -30,7 +30,9 @@ SELECT count() FROM t_async_inserts_flush;
 
 SYSTEM FLUSH ASYNC INSERT QUEUE;
 
-SELECT count() FROM system.asynchronous_inserts;
+SELECT count() FROM system.asynchronous_inserts
+WHERE database = currentDatabase() AND table = 't_async_inserts_flush';
+
 SELECT count() FROM t_async_inserts_flush;
 
 DROP TABLE t_async_inserts_flush;

tests/queries/0_stateless/02726_async_insert_flush_stress.sh

@@ -91,5 +91,5 @@ flush1 $TIMEOUT &
 wait
 
 ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH ASYNC INSERT QUEUE"
-${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.asynchronous_inserts"
+${CLICKHOUSE_CLIENT} -q "SELECT count() FROM system.asynchronous_inserts WHERE database = currentDatabase() AND table = 'async_inserts'"
 ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS async_inserts";