ClickHouse/tests/queries/0_stateless/01747_system_session_log_long.sh

#!/usr/bin/env bash

##################################################################################################
# Verify that login, logout, and login failure events are properly stored in system.session_log
# when different `IDENTIFIED BY` clauses are used on user.
#
# Make sure that system.session_log entries are non-empty and provide enough info on each event.
#
# Using multiple protocols
# * native TCP protocol with CH client
# * HTTP with CURL
# * MySQL - CH server accesses itself via mysql table function, query typically fails (unrelated)
#   but auth should be performed properly.
# * PostgreSQL - CH server accesses itself via postgresql table function (currently out of order).
# * gRPC - not done yet
#
# There is way to control how many time a query (e.g. via mysql table function) is retried
# and hence variable number of records in session_log. To mitigate this and simplify final query,
# each auth_type is tested for separate user. That way SELECT DISTINCT doesn't exclude log entries
# from different cases.
#
# All created users added to the ALL_USERNAMES and later cleaned up.
##################################################################################################

# To minimize amount of error context sent on failed queries when talking to CH via MySQL protocol.
export CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none

CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CURDIR"/../shell_config.sh

set -eu

# Since there is no way to cleanup system.session_log table,
# make sure that we can identify log entries from this test by a random user name.
readonly BASE_USERNAME="session_log_test_user_$(cat /dev/urandom | tr -cd 'a-f0-9' | head -c 32)"
readonly TMP_QUERY_FILE=$(mktemp /tmp/tmp_query.log.XXXXXX)
declare -a ALL_USERNAMES
ALL_USERNAMES+=("${BASE_USERNAME}")

function reportError()
{
    if [ -s "${TMP_QUERY_FILE}" ] ;
    then
        echo "!!!!!! ERROR ${CLICKHOUSE_CLIENT} ${*} --queries-file ${TMP_QUERY_FILE}" >&2
        echo "query:" >&2
        cat "${TMP_QUERY_FILE}" >&2
        rm -f "${TMP_QUERY_FILE}"
    fi
}

function executeQuery()
{
    ## Execute query (provided via heredoc or herestring) and print query in case of error.
    trap 'rm -f ${TMP_QUERY_FILE}; trap - ERR RETURN' RETURN
    # Since we want to report with current values supplied to this function call
    # shellcheck disable=SC2064
    trap "reportError $*" ERR

    cat - > "${TMP_QUERY_FILE}"
    ${CLICKHOUSE_CLIENT} "${@}" --queries-file "${TMP_QUERY_FILE}"
}

function cleanup()
{
    local usernames_to_cleanup
    usernames_to_cleanup="$(IFS=, ; echo "${ALL_USERNAMES[*]}")"
    executeQuery <<EOF
DROP USER IF EXISTS ${usernames_to_cleanup};
DROP SETTINGS PROFILE IF EXISTS session_log_test_profile;
DROP SETTINGS PROFILE IF EXISTS session_log_test_profile2;
DROP ROLE IF EXISTS session_log_test_role;
DROP ROLE IF EXISTS session_log_test_role2;
EOF
}

cleanup
trap "cleanup" EXIT

function executeQueryExpectError()
{
    cat - > "${TMP_QUERY_FILE}"
    ! ${CLICKHOUSE_CLIENT} "${@}" --multiquery --queries-file "${TMP_QUERY_FILE}" 2>&1 | tee -a ${TMP_QUERY_FILE}
}

function createUser()
{
    local auth_type="${1}"
    local username="${2}"
    local password="${3}"

    if [[ "${auth_type}" == "no_password" ]]
    then
        password=""

    elif [[ "${auth_type}" == "plaintext_password" ]]
    then
        password="${password}"

    elif [[ "${auth_type}" == "sha256_password" ]]
    then
        password="$(executeQuery <<< "SELECT hex(SHA256('${password}'))")"

    elif [[ "${auth_type}" == "double_sha1_password" ]]
    then
        password="$(executeQuery <<< "SELECT hex(SHA1(SHA1('${password}')))")"

    else
        echo "Invalid auth_type: ${auth_type}" >&2
        exit 1
    fi

    export RESULTING_PASS="${password}"
    if [ -n "${password}" ]
    then
        password="BY '${password}'"
    fi

    executeQuery <<EOF
DROP USER IF EXISTS '${username}';
CREATE USER '${username}' IDENTIFIED WITH ${auth_type} ${password};
EOF
    ALL_USERNAMES+=("${username}")
}

function testTCP()
{
    echo "TCP endpoint"

    local auth_type="${1}"
    local username="${2}"
    local password="${3}"

    # Loging\Logout
    if [[ -n "${password}" ]]
    then
        executeQuery -u "${username}" --password "${password}" <<< "SELECT 1 FORMAT Null;"
    else
        executeQuery -u "${username}" <<< "SELECT 1 FORMAT Null;" 
    fi

    # Wrong username
    executeQueryExpectError -u "invalid_${username}" \
        <<< "SELECT 1 Format Null" \
        | grep -Eq "Code: 516. .+ invalid_${username}: Authentication failed: password is incorrect or there is no user with such name"

    # Wrong password
    if [[ "${auth_type}" == "no_password" ]]
    then
        echo "TCP 'wrong password' case is skipped for ${auth_type}."
    else
        # user with `no_password` user is able to login with any password, so it makes sense to skip this testcase.
        executeQueryExpectError -u "${username}" --password  "invalid_${password}" \
            <<< "SELECT 1 Format Null" \
            | grep -Eq "Code: 516. .+ ${username}: Authentication failed: password is incorrect or there is no user with such name"
    fi
}

function testHTTPWithURL()
{
    local auth_type="${1}"
    local username="${2}"
    local password="${3}"
    local clickhouse_url="${4}"

    # Loging\Logout
    ${CLICKHOUSE_CURL} -sS "${clickhouse_url}" \
        -H "X-ClickHouse-User: ${username}" -H "X-ClickHouse-Key: ${password}" \
        -d 'SELECT 1 Format Null'

    # Wrong username
    ${CLICKHOUSE_CURL} -sS "${clickhouse_url}" \
        -H "X-ClickHouse-User: invalid_${username}" -H "X-ClickHouse-Key: ${password}" \
        -d 'SELECT 1 Format Null' \
        | grep -Eq "Code: 516. .+ invalid_${username}: Authentication failed: password is incorrect or there is no user with such name"

    # Wrong password
    if [[ "${auth_type}" == "no_password" ]]
    then
        echo "HTTP 'wrong password' case is skipped for ${auth_type}."
    else
        # user with `no_password` is able to login with any password, so it makes sense to skip this testcase.
        ${CLICKHOUSE_CURL} -sS "${clickhouse_url}" \
            -H "X-ClickHouse-User: ${username}" -H "X-ClickHouse-Key: invalid_${password}" \
            -d 'SELECT 1 Format Null' \
            | grep -Eq "Code: 516. .+ ${username}: Authentication failed: password is incorrect or there is no user with such name"
    fi
}

function testHTTP()
{
    echo "HTTP endpoint"
    testHTTPWithURL "${1}" "${2}" "${3}" "${CLICKHOUSE_URL}"
}

function testHTTPNamedSession()
{
    # echo "HTTP endpoint with named session"
    local HTTP_SESSION_ID
    HTTP_SESSION_ID="session_id_$(cat /dev/urandom | tr -cd 'a-f0-9' | head -c 32)"
    if [ -v CLICKHOUSE_URL_PARAMS ]
    then
        CLICKHOUSE_URL_WITH_SESSION_ID="${CLICKHOUSE_URL}&session_id=${HTTP_SESSION_ID}"
    else
        CLICKHOUSE_URL_WITH_SESSION_ID="${CLICKHOUSE_URL}?session_id=${HTTP_SESSION_ID}"
    fi

    testHTTPWithURL "${1}" "${2}" "${3}" "${CLICKHOUSE_URL_WITH_SESSION_ID}"
}

function testMySQL()
{
    echo "MySQL endpoint"
    local auth_type="${1}"
    local username="${2}"
    local password="${3}"

    trap "reportError" ERR

    # echo 'Loging\Logout'
    # sha256 auth is done differenctly for MySQL, so skip it for now.
    if [[ "${auth_type}" == "sha256_password" ]]
    then
        echo "MySQL 'successful login' case is skipped for ${auth_type}."
    else
        # CH is able to log into itself via MySQL protocol but query fails.
        executeQueryExpectError \
            <<< "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'numbers', '${username}', '${password}') LIMIT 1 \
            FORMAT NUll" \
            | grep -Eq "Code: 1000\. DB::Exception: .*"
    fi

    # echo 'Wrong username'
    executeQueryExpectError \
        <<< "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'numbers', 'invalid_${username}', '${password}') LIMIT 1 \
        FORMAT NUll" \
        | grep -Eq "Code: 1000\. DB::Exception: .* invalid_${username}"

    # echo 'Wrong password'
    if [[ "${auth_type}" == "no_password" ]]
    then
        echo "MySQL 'wrong password' case is skipped for ${auth_type}."
    else
        # user with `no_password` is able to login with any password, so it makes sense to skip this testcase.
        executeQueryExpectError \
            <<< "SELECT 1 FROM mysql('127.0.0.1:9004', 'system', 'numbers', '${username}', 'invalid_${password}') LIMIT 1 \
            FORMAT NUll" \
            | grep -Eq "Code: 1000\. DB::Exception: .* ${username}"
    fi
}

# function testPostgreSQL()
# {
#     local auth_type="${1}"
#
#     # Right now it is impossible to log into CH via PostgreSQL protocol without a password.
#     if [[ "${auth_type}" == "no_password" ]]
#     then
#         return 0
#     fi
#
#     # Loging\Logout
#     # CH is being able to log into itself via PostgreSQL protocol but query fails.
#     executeQueryExpectError \
#         <<< "SELECT 1 FROM postgresql('localhost:9005', 'system', 'numbers', '${username}', '${password}') LIMIT 1 FORMAT NUll" \
#         | grep -Eq "Code: 1001. DB::Exception: .* pqxx::broken_connection: .*"
#
#     # Wrong username
#     executeQueryExpectError \
#         <<< "SELECT 1 FROM postgresql('localhost:9005', 'system', 'numbers', 'invalid_${username}', '${password}') LIMIT 1 FORMAT NUll" \
#         | grep -Eq "Code: 1001. DB::Exception: .* pqxx::broken_connection: .*"
#
#     # Wrong password
#     executeQueryExpectError \
#         <<< "SELECT 1 FROM postgresql('localhost:9005', 'system', 'numbers', '${username}', 'invalid_${password}') LIMIT 1 FORMAT NUll" \
#         | grep -Eq "Code: 1001. DB::Exception: .* pqxx::broken_connection: .*"
# }

function runEndpointTests()
{
    local case_name="${1}"
    shift 1

    local auth_type="${1}"
    local username="${2}"
    local password="${3}"
    local setup_queries="${4:-}"

    echo 
    echo "#  ${auth_type} - ${case_name} "

    ${CLICKHOUSE_CLIENT} -q "SET log_comment='${username} ${auth_type} - ${case_name}';"
    if [[ -n "${setup_queries}" ]]
    then
        # echo "Executing setup queries: ${setup_queries}"
        echo "${setup_queries}" | executeQuery --multiquery
    fi

    testTCP "${auth_type}" "${username}" "${password}"
    testHTTP "${auth_type}" "${username}" "${password}"

    # testHTTPNamedSession "${auth_type}" "${username}" "${password}"
    testMySQL "${auth_type}" "${username}" "${password}"
    # testPostgreSQL "${auth_type}" "${username}" "${password}"
}

function testAsUserIdentifiedBy()
{
    local auth_type="${1}"
    local password="password"

    cleanup

    local username="${BASE_USERNAME}_${auth_type}_no_profiles_no_roles"
    createUser "${auth_type}" "${username}" "${password}"
    runEndpointTests "No profiles no roles" "${auth_type}" "${username}" "${RESULTING_PASS}"

    username="${BASE_USERNAME}_${auth_type}_two_profiles_no_roles"
    createUser "${auth_type}" "${username}" "${password}"
    runEndpointTests "Two profiles, no roles" "${auth_type}" "${username}" "${RESULTING_PASS}" "\
DROP SETTINGS PROFILE IF EXISTS session_log_test_profile;
DROP SETTINGS PROFILE IF EXISTS session_log_test_profile2;
CREATE PROFILE session_log_test_profile SETTINGS max_memory_usage=10000000 TO ${username};
CREATE PROFILE session_log_test_profile2 SETTINGS max_rows_to_transfer=1000 TO ${username};
"

    username="${BASE_USERNAME}_${auth_type}_two_profiles_two_roles"
    createUser "${auth_type}" "${username}" "${password}"
    runEndpointTests "Two profiles and two simple roles" "${auth_type}" "${username}" "${RESULTING_PASS}" "\
CREATE ROLE session_log_test_role;
GRANT session_log_test_role TO ${username};
CREATE ROLE session_log_test_role2 SETTINGS max_columns_to_read=100;
GRANT session_log_test_role2 TO ${username};
SET DEFAULT ROLE session_log_test_role, session_log_test_role2 TO ${username};
"
}

# to cut off previous runs
readonly start_time="$(executeQuery <<< 'SELECT now64(6);')"

# Special case: user and profile are both defined in XML
runEndpointTests "User with profile from XML" "no_password" "session_log_test_xml_user" ''

testAsUserIdentifiedBy "no_password"
testAsUserIdentifiedBy "plaintext_password"
testAsUserIdentifiedBy "sha256_password"
testAsUserIdentifiedBy "double_sha1_password"

executeQuery --multiquery <<EOF
SYSTEM FLUSH LOGS;

WITH
    now64(6) as n,
    toDateTime64('${start_time}', 3) as test_start_time
SELECT
    replaceAll(user, '${BASE_USERNAME}', '\${BASE_USERNAME}') as user_name,
    interface,
    type,
    if(count(*) > 1, 'many', toString(count(*))) -- do not rely on count value since MySQL does arbitrary number of retries
FROM
    system.session_log
WHERE
    (user LIKE '%session_log_test_xml_user%' OR user LIKE '%${BASE_USERNAME}%')
    AND
    event_time_microseconds >= test_start_time
GROUP BY
    user_name, interface, type
ORDER BY
    user_name, interface, type;
EOF