ClickHouse/tests/queries/0_stateless/01747_system_session_log_long.sh

370 lines
13 KiB
Bash
Raw Normal View History

#!/usr/bin/env bash
##################################################################################################
# Verify that login, logout, and login failure events are properly stored in system.session_log
# when different `IDENTIFIED BY` clauses are used on user.
#
# Make sure that system.session_log entries are non-empty and provide enough info on each event.
#
# Using multiple protocols
# * native TCP protocol with CH client
# * HTTP with CURL
# * MySQL - CH server accesses itself via mysql table function, query typically fails (unrelated)
# but auth should be performed properly.
# * PostgreSQL - CH server accesses itself via postgresql table function (currently out of order).
# * gRPC - not done yet
#
# There is way to control how many time a query (e.g. via mysql table function) is retried
# and hence variable number of records in session_log. To mitigate this and simplify final query,
# each auth_type is tested for separate user. That way SELECT DISTINCT doesn't exclude log entries
# from different cases.
#
# All created users added to the ALL_USERNAMES and later cleaned up.
##################################################################################################
# To minimize amount of error context sent on failed queries when talking to CH via MySQL protocol.
export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh
set -eu
# Since there is no way to cleanup system.session_log table,
# make sure that we can identify log entries from this test by a random user name.
readonly BASE_USERNAME="session_log_test_user_$(cat /dev/urandom | tr -cd 'a-f0-9' | head -c 32)"
readonly TMP_QUERY_FILE=$(mktemp /tmp/tmp_query.log.XXXXXX)
declare -a ALL_USERNAMES
ALL_USERNAMES+=("${BASE_USERNAME}")
function reportError()
{
if [ -s "${TMP_QUERY_FILE}" ] ;
then
echo "!!!!!! ERROR ${CLICKHOUSE_CLIENT} ${*} --queries-file ${TMP_QUERY_FILE}" >&2
echo "query:" >&2
cat "${TMP_QUERY_FILE}" >&2
rm -f "${TMP_QUERY_FILE}"
fi
}
function executeQuery()
{
## Execute query (provided via heredoc or herestring) and print query in case of error.
trap 'rm -f ${TMP_QUERY_FILE}; trap - ERR RETURN' RETURN
# Since we want to report with current values supplied to this function call
# shellcheck disable=SC2064
trap "reportError $*" ERR
cat - > "${TMP_QUERY_FILE}"
${CLICKHOUSE_CLIENT} "${@}" --queries-file "${TMP_QUERY_FILE}"
}
function cleanup()
{
local usernames_to_cleanup
usernames_to_cleanup="$(IFS=, ; echo "${ALL_USERNAMES[*]}")"
executeQuery <<EOF
DROP USER IF EXISTS ${usernames_to_cleanup};
DROP SETTINGS PROFILE IF EXISTS session_log_test_profile;
DROP SETTINGS PROFILE IF EXISTS session_log_test_profile2;
DROP ROLE IF EXISTS session_log_test_role;
DROP ROLE IF EXISTS session_log_test_role2;
EOF
}
cleanup
trap "cleanup" EXIT
function executeQueryExpectError()
{
cat - > "${TMP_QUERY_FILE}"
! ${CLICKHOUSE_CLIENT} "${@}" --multiquery --queries-file "${TMP_QUERY_FILE}" 2>&1 | tee -a ${TMP_QUERY_FILE}
}
function createUser()
{
local auth_type="${1}"
local username="${2}"
local password="${3}"
if [[ "${auth_type}" == "no_password" ]]
then
password=""
elif [[ "${auth_type}" == "plaintext_password" ]]
then
password="${password}"
elif [[ "${auth_type}" == "sha256_password" ]]
then
password="$(executeQuery <<< "SELECT hex(SHA256('${password}'))")"
elif [[ "${auth_type}" == "double_sha1_password" ]]
then
password="$(executeQuery <<< "SELECT hex(SHA1(SHA1('${password}')))")"
else
echo "Invalid auth_type: ${auth_type}" >&2
exit 1
fi
export RESULTING_PASS="${password}"
if [ -n "${password}" ]
then
password="BY '${password}'"
fi
executeQuery <<EOF
DROP USER IF EXISTS '${username}';
CREATE USER '${username}' IDENTIFIED WITH ${auth_type} ${password};
EOF
ALL_USERNAMES+=("${username}")
}
function testTCP()
{
echo "TCP endpoint"
local auth_type="${1}"
local username="${2}"
local password="${3}"
# Loging\Logout
if [[ -n "${password}" ]]
then
executeQuery -u "${username}" --password "${password}" <<< "SELECT 1 FORMAT Null;"
else
executeQuery -u "${username}" <<< "SELECT 1 FORMAT Null;"
fi
# Wrong username
executeQueryExpectError -u "invalid_${username}" \
<<< "SELECT 1 Format Null" \
| grep -Eq "Code: 516. .+ invalid_${username}: Authentication failed: password is incorrect or there is no user with such name"
# Wrong password
if [[ "${auth_type}" == "no_password" ]]
then
echo "TCP 'wrong password' case is skipped for ${auth_type}."
else
# user with `no_password` user is able to login with any password, so it makes sense to skip this testcase.
executeQueryExpectError -u "${username}" --password "invalid_${password}" \
<<< "SELECT 1 Format Null" \
| grep -Eq "Code: 516. .+ ${username}: Authentication failed: password is incorrect or there is no user with such name"
fi
}
function testHTTPWithURL()
{
local auth_type="${1}"
local username="${2}"
local password="${3}"
local clickhouse_url="${4}"
# Loging\Logout
${CLICKHOUSE_CURL} -sS "${clickhouse_url}" \
-H "X-ClickHouse-User: ${username}" -H "X-ClickHouse-Key: ${password}" \
-d 'SELECT 1 Format Null'
# Wrong username
${CLICKHOUSE_CURL} -sS "${clickhouse_url}" \
-H "X-ClickHouse-User: invalid_${username}" -H "X-ClickHouse-Key: ${password}" \
-d 'SELECT 1 Format Null' \
| grep -Eq "Code: 516. .+ invalid_${username}: Authentication failed: password is incorrect or there is no user with such name"
# Wrong password
if [[ "${auth_type}" == "no_password" ]]
then
echo "HTTP 'wrong password' case is skipped for ${auth_type}."
else
# user with `no_password` is able to login with any password, so it makes sense to skip this testcase.
${CLICKHOUSE_CURL} -sS "${clickhouse_url}" \
-H "X-ClickHouse-User: ${username}" -H "X-ClickHouse-Key: invalid_${password}" \
-d 'SELECT 1 Format Null' \
| grep -Eq "Code: 516. .+ ${username}: Authentication failed: password is incorrect or there is no user with such name"
fi
}
function testHTTP()
{
echo "HTTP endpoint"
testHTTPWithURL "${1}" "${2}" "${3}" "${CLICKHOUSE_URL}"
}
function testHTTPNamedSession()
{
# echo "HTTP endpoint with named session"
local HTTP_SESSION_ID
HTTP_SESSION_ID="session_id_$(cat /dev/urandom | tr -cd 'a-f0-9' | head -c 32)"
if [ -v CLICKHOUSE_URL_PARAMS ]
then
CLICKHOUSE_URL_WITH_SESSION_ID="${CLICKHOUSE_URL}&session_id=${HTTP_SESSION_ID}"
else
CLICKHOUSE_URL_WITH_SESSION_ID="${CLICKHOUSE_URL}?session_id=${HTTP_SESSION_ID}"
fi
testHTTPWithURL "${1}" "${2}" "${3}" "${CLICKHOUSE_URL_WITH_SESSION_ID}"
}
function testMySQL()
{
echo "MySQL endpoint"
local auth_type="${1}"
local username="${2}"
local password="${3}"
trap "reportError" ERR
# echo 'Loging\Logout'
# sha256 auth is done differenctly for MySQL, so skip it for now.
if [[ "${auth_type}" == "sha256_password" ]]
then
echo "MySQL 'successful login' case is skipped for ${auth_type}."
else
# CH is able to log into itself via MySQL protocol but query fails.
executeQueryExpectError \
<<< "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'numbers', '${username}', '${password}') LIMIT 1 \
FORMAT NUll" \
| grep -Eq "Code: 1000\. DB::Exception: .*"
fi
# echo 'Wrong username'
executeQueryExpectError \
<<< "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'numbers', 'invalid_${username}', '${password}') LIMIT 1 \
FORMAT NUll" \
| grep -Eq "Code: 1000\. DB::Exception: .* invalid_${username}"
# echo 'Wrong password'
if [[ "${auth_type}" == "no_password" ]]
then
echo "MySQL 'wrong password' case is skipped for ${auth_type}."
else
# user with `no_password` is able to login with any password, so it makes sense to skip this testcase.
executeQueryExpectError \
<<< "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'numbers', '${username}', 'invalid_${password}') LIMIT 1 \
FORMAT NUll" \
| grep -Eq "Code: 1000\. DB::Exception: .* ${username}"
fi
}
# function testPostgreSQL()
# {
# local auth_type="${1}"
#
# # Right now it is impossible to log into CH via PostgreSQL protocol without a password.
# if [[ "${auth_type}" == "no_password" ]]
# then
# return 0
# fi
#
# # Loging\Logout
# # CH is being able to log into itself via PostgreSQL protocol but query fails.
# executeQueryExpectError \
# <<< "SELECT 1 FROM postgresql('localhost:9005', 'system', 'numbers', '${username}', '${password}') LIMIT 1 FORMAT NUll" \
# | grep -Eq "Code: 1001. DB::Exception: .* pqxx::broken_connection: .*"
#
# # Wrong username
# executeQueryExpectError \
# <<< "SELECT 1 FROM postgresql('localhost:9005', 'system', 'numbers', 'invalid_${username}', '${password}') LIMIT 1 FORMAT NUll" \
# | grep -Eq "Code: 1001. DB::Exception: .* pqxx::broken_connection: .*"
#
# # Wrong password
# executeQueryExpectError \
# <<< "SELECT 1 FROM postgresql('localhost:9005', 'system', 'numbers', '${username}', 'invalid_${password}') LIMIT 1 FORMAT NUll" \
# | grep -Eq "Code: 1001. DB::Exception: .* pqxx::broken_connection: .*"
# }
function runEndpointTests()
{
local case_name="${1}"
shift 1
local auth_type="${1}"
local username="${2}"
local password="${3}"
local setup_queries="${4:-}"
echo
echo "# ${auth_type} - ${case_name} "
${CLICKHOUSE_CLIENT} -q "SET log_comment='${username} ${auth_type} - ${case_name}';"
if [[ -n "${setup_queries}" ]]
then
# echo "Executing setup queries: ${setup_queries}"
echo "${setup_queries}" | executeQuery --multiquery
fi
testTCP "${auth_type}" "${username}" "${password}"
testHTTP "${auth_type}" "${username}" "${password}"
# testHTTPNamedSession "${auth_type}" "${username}" "${password}"
testMySQL "${auth_type}" "${username}" "${password}"
# testPostgreSQL "${auth_type}" "${username}" "${password}"
}
function testAsUserIdentifiedBy()
{
local auth_type="${1}"
local password="password"
cleanup
local username="${BASE_USERNAME}_${auth_type}_no_profiles_no_roles"
createUser "${auth_type}" "${username}" "${password}"
runEndpointTests "No profiles no roles" "${auth_type}" "${username}" "${RESULTING_PASS}"
username="${BASE_USERNAME}_${auth_type}_two_profiles_no_roles"
createUser "${auth_type}" "${username}" "${password}"
runEndpointTests "Two profiles, no roles" "${auth_type}" "${username}" "${RESULTING_PASS}" "\
DROP SETTINGS PROFILE IF EXISTS session_log_test_profile;
DROP SETTINGS PROFILE IF EXISTS session_log_test_profile2;
CREATE PROFILE session_log_test_profile SETTINGS max_memory_usage=10000000 TO ${username};
CREATE PROFILE session_log_test_profile2 SETTINGS max_rows_to_transfer=1000 TO ${username};
"
username="${BASE_USERNAME}_${auth_type}_two_profiles_two_roles"
createUser "${auth_type}" "${username}" "${password}"
runEndpointTests "Two profiles and two simple roles" "${auth_type}" "${username}" "${RESULTING_PASS}" "\
CREATE ROLE session_log_test_role;
GRANT session_log_test_role TO ${username};
CREATE ROLE session_log_test_role2 SETTINGS max_columns_to_read=100;
GRANT session_log_test_role2 TO ${username};
SET DEFAULT ROLE session_log_test_role, session_log_test_role2 TO ${username};
"
}
# to cut off previous runs
readonly start_time="$(executeQuery <<< 'SELECT now64(6);')"
# Special case: user and profile are both defined in XML
runEndpointTests "User with profile from XML" "no_password" "session_log_test_xml_user" ''
testAsUserIdentifiedBy "no_password"
testAsUserIdentifiedBy "plaintext_password"
testAsUserIdentifiedBy "sha256_password"
testAsUserIdentifiedBy "double_sha1_password"
executeQuery --multiquery <<EOF
SYSTEM FLUSH LOGS;
WITH
now64(6) as n,
toDateTime64('${start_time}', 3) as test_start_time
SELECT
replaceAll(user, '${BASE_USERNAME}', '\${BASE_USERNAME}') as user_name,
interface,
type,
if(count(*) > 1, 'many', toString(count(*))) -- do not rely on count value since MySQL does arbitrary number of retries
FROM
system.session_log
WHERE
(user LIKE '%session_log_test_xml_user%' OR user LIKE '%${BASE_USERNAME}%')
AND
event_time_microseconds >= test_start_time
GROUP BY
user_name, interface, type
ORDER BY
user_name, interface, type;
EOF