thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into no_background_pool_no_more

Browse Source
This commit is contained in:
alesapin 2020-10-23 10:28:23 +03:00
commit 6df68d6c80
22 changed files with 731 additions and 300 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
docker/test/testflows/runner/Dockerfile
View File

@ -35,7 +35,7 @@ RUN apt-get update \
ENV TZ=Europe/Moscow
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
RUN pip3 install urllib3 testflows==1.6.48 docker-compose docker dicttoxml kazoo tzlocal
RUN pip3 install urllib3 testflows==1.6.57 docker-compose docker dicttoxml kazoo tzlocal
ENV DOCKER_CHANNEL stable
ENV DOCKER_VERSION 17.09.1-ce
@ -72,5 +72,5 @@ RUN set -x \
VOLUME /var/lib/docker
EXPOSE 2375
ENTRYPOINT ["dockerd-entrypoint.sh"]
CMD ["sh", "-c", "python3 regression.py --no-color --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json"]
CMD ["sh", "-c", "python3 regression.py --no-color -o classic --local --clickhouse-binary-path ${CLICKHOUSE_TESTS_SERVER_BIN_PATH} --log test.log ${TESTFLOWS_OPTS}; cat test.log | tfs report results --format json > results.json"]

1
docs/en/introduction/adopters.md
View File

@ -77,6 +77,7 @@ toc_title: Adopters
| <a href="https://rambler.ru" class="favicon">Rambler</a> | Internet services | Analytics | — | — | [Talk in Russian, April 2018](https://medium.com/@ramblertop/разработка-api-clickhouse-для-рамблер-топ-100-f4c7e56f3141) |
| <a href="https://retell.cc/" class="favicon">Retell</a> | Speech synthesis | Analytics | — | — | [Blog Article, August 2020](https://vc.ru/services/153732-kak-sozdat-audiostati-na-vashem-sayte-i-zachem-eto-nuzhno) |
| <a href="https://rspamd.com/" class="favicon">Rspamd</a> | Antispam | Analytics | — | — | [Official Website](https://rspamd.com/doc/modules/clickhouse.html) |
| <a href="https://rusiem.com/en" class="favicon">RuSIEM</a> | SIEM | Main Product | — | — | [Official Website](https://rusiem.com/en/products/architecture) |
| <a href="https://www.s7.ru" class="favicon">S7 Airlines</a> | Airlines | Metrics, Logging | — | — | [Talk in Russian, March 2019](https://www.youtube.com/watch?v=nwG68klRpPg&t=15s) |
| <a href="https://www.scireum.de/" class="favicon">scireum GmbH</a> | e-Commerce | Main product | — | — | [Talk in German, February 2020](https://www.youtube.com/watch?v=7QWAn5RbyR4) |
| <a href="https://segment.com/" class="favicon">Segment</a> | Data processing | Main product | 9 * i3en.3xlarge nodes 7.5TB NVME SSDs, 96GB Memory, 12 vCPUs | — | [Slides, 2019](https://slides.com/abraithwaite/segment-clickhouse) |

2
docs/en/sql-reference/syntax.md
View File

@ -87,7 +87,7 @@ In string literals, you need to escape at least `'` and `\`. Single quotes can b
### Compound {#compound}
Arrays are constructed with square brackets `[1, 2, 3]`. Nuples are constructed with round brackets `(1, 'Hello, world!', 2)`.
Arrays are constructed with square brackets `[1, 2, 3]`. Tuples are constructed with round brackets `(1, 'Hello, world!', 2)`.
Technically these are not literals, but expressions with the array creation operator and the tuple creation operator, respectively.
An array must consist of at least one item, and a tuple must have at least two items.
There’s a separate case when tuples appear in the `IN` clause of a `SELECT` query. Query results can include tuples, but tuples can’t be saved to a database (except of tables with [Memory](../engines/table-engines/special/memory.md) engine).

84
programs/local/LocalServer.cpp
View File

@ -57,8 +57,8 @@ LocalServer::LocalServer() = default;
LocalServer::~LocalServer()
{
if (context)
context->shutdown(); /// required for properly exception handling
if (global_context)
global_context->shutdown(); /// required for properly exception handling
}
@ -95,9 +95,9 @@ void LocalServer::initialize(Poco::Util::Application & self)
}
}
void LocalServer::applyCmdSettings()
void LocalServer::applyCmdSettings(Context & context)
{
context->applySettingsChanges(cmd_settings.changes());
context.applySettingsChanges(cmd_settings.changes());
}
/// If path is specified and not empty, will try to setup server environment and load existing metadata
@ -151,8 +151,12 @@ void LocalServer::tryInitPath()
if (path.back() != '/')
path += '/';
context->setPath(path);
context->setUserFilesPath(""); // user's files are everywhere
global_context->setPath(path);
global_context->setTemporaryStorage(path + "tmp");
global_context->setFlagsPath(path + "flags");
global_context->setUserFilesPath(""); // user's files are everywhere
}
@ -186,9 +190,9 @@ try
}
shared_context = Context::createShared();
context = std::make_unique<Context>(Context::createGlobal(shared_context.get()));
context->makeGlobalContext();
context->setApplicationType(Context::ApplicationType::LOCAL);
global_context = std::make_unique<Context>(Context::createGlobal(shared_context.get()));
global_context->makeGlobalContext();
global_context->setApplicationType(Context::ApplicationType::LOCAL);
tryInitPath();
std::optional<StatusFile> status;
@ -210,32 +214,32 @@ try
/// Maybe useless
if (config().has("macros"))
context->setMacros(std::make_unique<Macros>(config(), "macros", log));
global_context->setMacros(std::make_unique<Macros>(config(), "macros", log));
/// Skip networking
/// Sets external authenticators config (LDAP).
context->setExternalAuthenticatorsConfig(config());
global_context->setExternalAuthenticatorsConfig(config());
setupUsers();
/// Limit on total number of concurrently executing queries.
/// There is no need for concurrent queries, override max_concurrent_queries.
context->getProcessList().setMaxSize(0);
global_context->getProcessList().setMaxSize(0);
/// Size of cache for uncompressed blocks. Zero means disabled.
size_t uncompressed_cache_size = config().getUInt64("uncompressed_cache_size", 0);
if (uncompressed_cache_size)
context->setUncompressedCache(uncompressed_cache_size);
global_context->setUncompressedCache(uncompressed_cache_size);
/// Size of cache for marks (index of MergeTree family of tables). It is necessary.
/// Specify default value for mark_cache_size explicitly!
size_t mark_cache_size = config().getUInt64("mark_cache_size", 5368709120);
if (mark_cache_size)
context->setMarkCache(mark_cache_size);
global_context->setMarkCache(mark_cache_size);
/// Load global settings from default_profile and system_profile.
context->setDefaultProfiles(config());
global_context->setDefaultProfiles(config());
/** Init dummy default DB
* NOTE: We force using isolated default database to avoid conflicts with default database from server environment
@ -243,34 +247,34 @@ try
* if such tables will not be dropped, clickhouse-server will not be able to load them due to security reasons.
*/
std::string default_database = config().getString("default_database", "_local");
DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, *context));
context->setCurrentDatabase(default_database);
applyCmdOptions();
DatabaseCatalog::instance().attachDatabase(default_database, std::make_shared<DatabaseMemory>(default_database, *global_context));
global_context->setCurrentDatabase(default_database);
applyCmdOptions(*global_context);
String path = context->getPath();
String path = global_context->getPath();
if (!path.empty())
{
/// Lock path directory before read
status.emplace(context->getPath() + "status", StatusFile::write_full_info);
status.emplace(global_context->getPath() + "status", StatusFile::write_full_info);
LOG_DEBUG(log, "Loading metadata from {}", path);
Poco::File(path + "data/").createDirectories();
Poco::File(path + "metadata/").createDirectories();
loadMetadataSystem(*context);
attachSystemTables(*context);
loadMetadata(*context);
loadMetadataSystem(*global_context);
attachSystemTables(*global_context);
loadMetadata(*global_context);
DatabaseCatalog::instance().loadDatabases();
LOG_DEBUG(log, "Loaded metadata.");
}
else
{
attachSystemTables(*context);
attachSystemTables(*global_context);
}
processQueries();
context->shutdown();
context.reset();
global_context->shutdown();
global_context.reset();
status.reset();
cleanup();
@ -323,7 +327,7 @@ void LocalServer::processQueries()
String initial_create_query = getInitialCreateTableQuery();
String queries_str = initial_create_query + config().getRawString("query");
const auto & settings = context->getSettingsRef();
const auto & settings = global_context->getSettingsRef();
std::vector<String> queries;
auto parse_res = splitMultipartQuery(queries_str, queries, settings.max_query_size, settings.max_parser_depth);
@ -331,15 +335,19 @@ void LocalServer::processQueries()
if (!parse_res.second)
throw Exception("Cannot parse and execute the following part of query: " + String(parse_res.first), ErrorCodes::SYNTAX_ERROR);
context->makeSessionContext();
context->makeQueryContext();
/// we can't mutate global global_context (can lead to races, as it was already passed to some background threads)
/// so we can't reuse it safely as a query context and need a copy here
auto context = Context(*global_context);
context->setUser("default", "", Poco::Net::SocketAddress{});
context->setCurrentQueryId("");
applyCmdSettings();
context.makeSessionContext();
context.makeQueryContext();
context.setUser("default", "", Poco::Net::SocketAddress{});
context.setCurrentQueryId("");
applyCmdSettings(context);
/// Use the same query_id (and thread group) for all queries
CurrentThread::QueryScope query_scope_holder(*context);
CurrentThread::QueryScope query_scope_holder(context);
bool echo_queries = config().hasOption("echo") || config().hasOption("verbose");
std::exception_ptr exception;
@ -358,7 +366,7 @@ void LocalServer::processQueries()
try
{
executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, *context, {});
executeQuery(read_buf, write_buf, /* allow_into_outfile = */ true, context, {});
}
catch (...)
{
@ -423,7 +431,7 @@ void LocalServer::setupUsers()
}
if (users_config)
context->setUsersConfig(users_config);
global_context->setUsersConfig(users_config);
else
throw Exception("Can't load config for users", ErrorCodes::CANNOT_LOAD_CONFIG);
}
@ -577,10 +585,10 @@ void LocalServer::init(int argc, char ** argv)
argsToConfig(arguments, config(), 100);
}
void LocalServer::applyCmdOptions()
void LocalServer::applyCmdOptions(Context & context)
{
context->setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
applyCmdSettings();
context.setDefaultFormat(config().getString("output-format", config().getString("format", "TSV")));
applyCmdSettings(context);
}
}

6
programs/local/LocalServer.h
View File

@ -36,15 +36,15 @@ private:
std::string getInitialCreateTableQuery();
void tryInitPath();
void applyCmdOptions();
void applyCmdSettings();
void applyCmdOptions(Context & context);
void applyCmdSettings(Context & context);
void processQueries();
void setupUsers();
void cleanup();
protected:
SharedContextHolder shared_context;
std::unique_ptr<Context> context;
std::unique_ptr<Context> global_context;
/// Settings specified via command line args
Settings cmd_settings;

15
src/Access/AccessControlManager.cpp
View File

@ -137,7 +137,6 @@ AccessControlManager::AccessControlManager()
AccessControlManager::~AccessControlManager() = default;
void AccessControlManager::setUsersConfig(const Poco::Util::AbstractConfiguration & users_config_)
{
auto storages = getStoragesPtr();
@ -163,6 +162,7 @@ void AccessControlManager::addUsersConfigStorage(const String & storage_name_, c
auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function);
new_storage->setConfig(users_config_);
addStorage(new_storage);
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
}
void AccessControlManager::addUsersConfigStorage(
@ -195,6 +195,7 @@ void AccessControlManager::addUsersConfigStorage(
auto new_storage = std::make_shared<UsersConfigAccessStorage>(storage_name_, check_setting_name_function);
new_storage->load(users_config_path_, include_from_path_, preprocessed_dir_, get_zookeeper_function_);
addStorage(new_storage);
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
}
void AccessControlManager::reloadUsersConfigs()
@ -238,7 +239,9 @@ void AccessControlManager::addDiskStorage(const String & storage_name_, const St
}
}
}
addStorage(std::make_shared<DiskAccessStorage>(storage_name_, directory_, readonly_));
auto new_storage = std::make_shared<DiskAccessStorage>(storage_name_, directory_, readonly_);
addStorage(new_storage);
LOG_DEBUG(getLogger(), "Added {} access storage '{}', path: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getPath());
}
@ -250,13 +253,17 @@ void AccessControlManager::addMemoryStorage(const String & storage_name_)
if (auto memory_storage = typeid_cast<std::shared_ptr<MemoryAccessStorage>>(storage))
return;
}
addStorage(std::make_shared<MemoryAccessStorage>(storage_name_));
auto new_storage = std::make_shared<MemoryAccessStorage>(storage_name_);
addStorage(new_storage);
LOG_DEBUG(getLogger(), "Added {} access storage '{}'", String(new_storage->getStorageType()), new_storage->getStorageName());
}
void AccessControlManager::addLDAPStorage(const String & storage_name_, const Poco::Util::AbstractConfiguration & config_, const String & prefix_)
{
addStorage(std::make_shared<LDAPAccessStorage>(storage_name_, this, config_, prefix_));
auto new_storage = std::make_shared<LDAPAccessStorage>(storage_name_, this, config_, prefix_);
addStorage(new_storage);
LOG_DEBUG(getLogger(), "Added {} access storage '{}', LDAP server name: {}", String(new_storage->getStorageType()), new_storage->getStorageName(), new_storage->getLDAPServerName());
}

6
src/Access/LDAPAccessStorage.cpp
View File

@ -29,6 +29,12 @@ LDAPAccessStorage::LDAPAccessStorage(const String & storage_name_, AccessControl
}
String LDAPAccessStorage::getLDAPServerName() const
{
return ldap_server;
}
void LDAPAccessStorage::setConfiguration(AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix)
{
std::scoped_lock lock(mutex);

2
src/Access/LDAPAccessStorage.h
View File

@ -32,6 +32,8 @@ public:
explicit LDAPAccessStorage(const String & storage_name_, AccessControlManager * access_control_manager_, const Poco::Util::AbstractConfiguration & config, const String & prefix);
virtual ~LDAPAccessStorage() override = default;
String getLDAPServerName() const;
public: // IAccessStorage implementations.
virtual const char * getStorageType() const override;
virtual String getStorageParamsJSON() const override;

0
tests/queries/0_stateless/01527_clickhouse_local_optimize.reference Normal file
View File

13
tests/queries/0_stateless/01527_clickhouse_local_optimize.sh Executable file
View File

@ -0,0 +1,13 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
WORKING_FOLDER_01527="${CLICKHOUSE_TMP}/01527_clickhouse_local_optimize"
rm -rf "${WORKING_FOLDER_01527}"
mkdir -p "${WORKING_FOLDER_01527}"
# OPTIMIZE was crashing due to lack of temporary volume in local
${CLICKHOUSE_LOCAL} --query "drop database if exists d; create database d; create table d.t engine MergeTree order by a as select 1 a; optimize table d.t final" -- --path="${WORKING_FOLDER_01527}"
rm -rf "${WORKING_FOLDER_01527}"

19
tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.reference Normal file
View File

@ -0,0 +1,19 @@
Option 1. Prepare parts from from table with Engine=File defined in metadata, read from an arbitrary path
1 2020-01-01 String
2 2020-02-02 Another string
3 2020-03-03 One more string
4 2020-01-02 String for first partition
Option 2. Prepare parts from from table with Engine=File defined in metadata, read from stdin (pipe)
11 2020-01-01 String
12 2020-02-02 Another string
13 2020-03-03 One more string
14 2020-01-02 String for first partition
Option 3. Prepare parts from from table with Engine=File defined via command line, read from stdin (pipe)
21 2020-01-01 String
22 2020-02-02 Another string
23 2020-03-03 One more string
24 2020-01-02 String for first partition
Possibility to run optimize on prepared parts before sending parts to server
202001 1
202002 1
202003 1

83
tests/queries/0_stateless/01528_clickhouse_local_prepare_parts.sh Executable file
View File

@ -0,0 +1,83 @@
#!/usr/bin/env bash
CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "$CURDIR"/../shell_config.sh
WORKING_FOLDER_01528="${CLICKHOUSE_TMP}/01528_clickhouse_local_prepare_parts"
rm -rf "${WORKING_FOLDER_01528}"
mkdir -p "${WORKING_FOLDER_01528}/metadata/local"
## Checks scenario of preparing parts offline by clickhouse-local
## that is the metadata for the table we want to fill
## schema should match the schema of the table from server
## (the easiest way is just to copy it from the server)
cat <<EOF > "${WORKING_FOLDER_01528}/metadata/local/test.sql"
ATTACH TABLE local.test (id UInt64, d Date, s String) Engine=MergeTree ORDER BY id PARTITION BY toYYYYMM(d);
EOF
#################
echo "Option 1. Prepare parts from from table with Engine=File defined in metadata, read from an arbitrary path"
## Source file:
cat <<EOF > "${WORKING_FOLDER_01528}/data.csv"
1,2020-01-01,"String"
2,2020-02-02,"Another string"
3,2020-03-03,"One more string"
4,2020-01-02,"String for first partition"
EOF
## metadata written into file
cat <<EOF > "${WORKING_FOLDER_01528}/metadata/local/data_csv.sql"
ATTACH TABLE local.data_csv (id UInt64, d Date, s String) Engine=File(CSV, '${WORKING_FOLDER_01528}/data.csv');
EOF
## feed the table
${CLICKHOUSE_LOCAL} --query "INSERT INTO local.test SELECT * FROM local.data_csv;" -- --path="${WORKING_FOLDER_01528}"
## check the parts were created
${CLICKHOUSE_LOCAL} --query "SELECT * FROM local.test WHERE id < 10 ORDER BY id;" -- --path="${WORKING_FOLDER_01528}"
#################
echo "Option 2. Prepare parts from from table with Engine=File defined in metadata, read from stdin (pipe)"
cat <<EOF > "${WORKING_FOLDER_01528}/metadata/local/stdin.sql"
ATTACH TABLE local.stdin (id UInt64, d Date, s String) Engine=File(CSV, stdin);
EOF
cat <<EOF | ${CLICKHOUSE_LOCAL} --query "INSERT INTO local.test SELECT * FROM local.stdin;" -- --path="${WORKING_FOLDER_01528}"
11,2020-01-01,"String"
12,2020-02-02,"Another string"
13,2020-03-03,"One more string"
14,2020-01-02,"String for first partition"
EOF
${CLICKHOUSE_LOCAL} --query "SELECT * FROM local.test WHERE id BETWEEN 10 AND 19 ORDER BY id;" -- --path="${WORKING_FOLDER_01528}"
#################
echo "Option 3. Prepare parts from from table with Engine=File defined via command line, read from stdin (pipe)"
cat <<EOF | ${CLICKHOUSE_LOCAL} --query "INSERT INTO local.test SELECT * FROM table;" -S "id UInt64, d Date, s String" --input-format=CSV -- --path="${WORKING_FOLDER_01528}"
21,2020-01-01,"String"
22,2020-02-02,"Another string"
23,2020-03-03,"One more string"
24,2020-01-02,"String for first partition"
EOF
${CLICKHOUSE_LOCAL} --query "SELECT * FROM local.test WHERE id BETWEEN 20 AND 29 ORDER BY id;" -- --path="${WORKING_FOLDER_01528}"
#################
echo "Possibility to run optimize on prepared parts before sending parts to server"
${CLICKHOUSE_LOCAL} --query "OPTIMIZE TABLE local.test FINAL;" -- --path="${WORKING_FOLDER_01528}"
# ensure we have one part per partition
${CLICKHOUSE_LOCAL} --query "SELECT toYYYYMM(d) m, uniqExact(_part) FROM local.test GROUP BY m ORDER BY m" -- --path="${WORKING_FOLDER_01528}"
# cleanup
rm -rf "${WORKING_FOLDER_01528}"

62
tests/testflows/ldap/authentication/tests/authentications.py
View File

@ -28,6 +28,8 @@ servers = {
@TestStep(When)
@Name("I login as {username} and execute query")
def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True):
"""Execute query as some user.
"""
self.context.node.query("SELECT 1",
settings=[("user", username), ("password", password)],
exitcode=exitcode or 0,
@ -35,7 +37,8 @@ def login_and_execute_query(self, username, password, exitcode=None, message=Non
@TestScenario
def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None, exitcode=None, message=None, rbac=False):
"""Add user to LDAP and ClickHouse and then try to login."""
"""Add user to LDAP and ClickHouse and then try to login.
"""
self.context.ldap_node = self.context.cluster.node(server)
if ch_user is None:
@ -60,7 +63,8 @@ def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None
RQ_SRS_007_LDAP_Authentication_Parallel_ValidAndInvalid("1.0")
)
def parallel_login(self, server, user_count=10, timeout=200, rbac=False):
"""Check that login of valid and invalid LDAP authenticated users works in parallel."""
"""Check that login of valid and invalid LDAP authenticated users works in parallel.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
@ -114,7 +118,8 @@ def parallel_login(self, server, user_count=10, timeout=200, rbac=False):
RQ_SRS_007_LDAP_Authentication_Invalid_DeletedUser("1.0")
)
def login_after_user_is_deleted_from_ldap(self, server, rbac=False):
"""Check that login fails after user is deleted from LDAP."""
"""Check that login fails after user is deleted from LDAP.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
@ -146,7 +151,8 @@ def login_after_user_is_deleted_from_ldap(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_PasswordChanged("1.0")
)
def login_after_user_password_changed_in_ldap(self, server, rbac=False):
"""Check that login fails after user password is changed in LDAP."""
"""Check that login fails after user password is changed in LDAP.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
@ -182,7 +188,8 @@ def login_after_user_password_changed_in_ldap(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_UsernameChanged("1.0")
)
def login_after_user_cn_changed_in_ldap(self, server, rbac=False):
"""Check that login fails after user cn is changed in LDAP."""
"""Check that login fails after user cn is changed in LDAP.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
new_user = None
@ -215,7 +222,8 @@ def login_after_user_cn_changed_in_ldap(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_LDAPServerRestart("1.0")
)
def login_after_ldap_server_is_restarted(self, server, timeout=60, rbac=False):
"""Check that login succeeds after LDAP server is restarted."""
"""Check that login succeeds after LDAP server is restarted.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
@ -250,7 +258,8 @@ def login_after_ldap_server_is_restarted(self, server, timeout=60, rbac=False):
RQ_SRS_007_LDAP_Authentication_ClickHouseServerRestart("1.0")
)
def login_after_clickhouse_server_is_restarted(self, server, timeout=60, rbac=False):
"""Check that login succeeds after ClickHouse server is restarted."""
"""Check that login succeeds after ClickHouse server is restarted.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
@ -285,7 +294,8 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=60, rbac=Fa
RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0")
)
def valid_username_with_valid_empty_password(self, server, rbac=False):
"""Check that we can't login using valid username that has empty password."""
"""Check that we can't login using valid username that has empty password.
"""
user = {"cn": "empty_password", "userpassword": ""}
exitcode = 4
message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name"
@ -298,7 +308,8 @@ def valid_username_with_valid_empty_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_Password_Empty("1.0")
)
def valid_username_and_invalid_empty_password(self, server, rbac=False):
"""Check that we can't login using valid username but invalid empty password."""
"""Check that we can't login using valid username but invalid empty password.
"""
username = "user_non_empty_password"
user = {"cn": username, "userpassword": username}
login = {"password": ""}
@ -313,7 +324,8 @@ def valid_username_and_invalid_empty_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_Valid("1.0")
)
def valid_username_and_password(self, server, rbac=False):
"""Check that we can login using valid username and password."""
"""Check that we can login using valid username and password.
"""
username = "valid_username_and_password"
user = {"cn": username, "userpassword": username}
@ -326,7 +338,8 @@ def valid_username_and_password(self, server, rbac=False):
)
def valid_username_and_password_invalid_server(self, server=None, rbac=False):
"""Check that we can't login using valid username and valid
password but for a different server."""
password but for a different server.
"""
self.context.ldap_node = self.context.cluster.node("openldap1")
user = {"username": "user2", "userpassword": "user2", "server": "openldap1"}
@ -344,7 +357,8 @@ def valid_username_and_password_invalid_server(self, server=None, rbac=False):
RQ_SRS_007_LDAP_Configuration_User_Name_Long("1.0")
)
def valid_long_username_and_short_password(self, server, rbac=False):
"""Check that we can login using valid very long username and short password."""
"""Check that we can login using valid very long username and short password.
"""
username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890"
user = {"cn": username, "userpassword": "long_username"}
@ -355,7 +369,8 @@ def valid_long_username_and_short_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_Invalid("1.0")
)
def invalid_long_username_and_valid_short_password(self, server, rbac=False):
"""Check that we can't login using slightly invalid long username but valid password."""
"""Check that we can't login using slightly invalid long username but valid password.
"""
username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890"
user = {"cn": username, "userpassword": "long_username"}
login = {"username": f"{username}?"}
@ -371,7 +386,8 @@ def invalid_long_username_and_valid_short_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_Password_Long("1.0")
)
def valid_short_username_and_long_password(self, server, rbac=False):
"""Check that we can login using valid short username with very long password."""
"""Check that we can login using valid short username with very long password.
"""
username = "long_password"
user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"}
add_user_to_ldap_and_login(user=user, server=server, rbac=rbac)
@ -381,7 +397,8 @@ def valid_short_username_and_long_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_Invalid("1.0")
)
def valid_short_username_and_invalid_long_password(self, server, rbac=False):
"""Check that we can't login using valid short username and invalid long password."""
"""Check that we can't login using valid short username and invalid long password.
"""
username = "long_password"
user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"}
login = {"password": user["userpassword"] + "1"}
@ -396,7 +413,8 @@ def valid_short_username_and_invalid_long_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_Invalid("1.0")
)
def valid_username_and_invalid_password(self, server, rbac=False):
"""Check that we can't login using valid username and invalid password."""
"""Check that we can't login using valid username and invalid password.
"""
username = "valid_username_and_invalid_password"
user = {"cn": username, "userpassword": username}
login = {"password": user["userpassword"] + "1"}
@ -411,7 +429,8 @@ def valid_username_and_invalid_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_Invalid("1.0")
)
def invalid_username_and_valid_password(self, server, rbac=False):
"""Check that we can't login using slightly invalid username but valid password."""
"""Check that we can't login using slightly invalid username but valid password.
"""
username = "invalid_username_and_valid_password"
user = {"cn": username, "userpassword": username}
login = {"username": user["cn"] + "1"}
@ -428,7 +447,8 @@ def invalid_username_and_valid_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Configuration_User_Name_UTF8("1.0")
)
def valid_utf8_username_and_ascii_password(self, server, rbac=False):
"""Check that we can login using valid utf-8 username with ascii password."""
"""Check that we can login using valid utf-8 username with ascii password.
"""
username = "utf8_username_Gãńdåłf_Thê_Gręât"
user = {"cn": username, "userpassword": "utf8_username"}
@ -440,7 +460,8 @@ def valid_utf8_username_and_ascii_password(self, server, rbac=False):
RQ_SRS_007_LDAP_Authentication_Password_UTF8("1.0")
)
def valid_ascii_username_and_utf8_password(self, server, rbac=False):
"""Check that we can login using valid ascii username with utf-8 password."""
"""Check that we can login using valid ascii username with utf-8 password.
"""
username = "utf8_password"
user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"}
@ -449,7 +470,8 @@ def valid_ascii_username_and_utf8_password(self, server, rbac=False):
@TestScenario
def empty_username_and_empty_password(self, server=None, rbac=False):
"""Check that we can login using empty username and empty password as
it will use the default user and that has an empty password."""
it will use the default user and that has an empty password.
"""
login_and_execute_query(username="", password="")
@TestOutline(Feature)

2
tests/testflows/ldap/authentication/tests/common.py
View File

@ -95,6 +95,8 @@ def add_config(config, timeout=20, restart=False):
if exitcode == 0:
break
time.sleep(1)
if settings.debug:
node.command(f"cat /var/lib/clickhouse/preprocessed_configs/{config.preprocessed_name}")
assert exitcode == 0, error()
def wait_for_config_to_be_loaded():

17
tests/testflows/ldap/authentication/tests/connections.py
View File

@ -98,7 +98,8 @@ def starttls_with_custom_port(self):
login(servers, *users)
def tls_connection(enable_tls, tls_require_cert):
"""Try to login using LDAP user authentication over a TLS connection."""
"""Try to login using LDAP user authentication over a TLS connection.
"""
servers = {
"openldap2": {
"host": "openldap2",
@ -152,7 +153,8 @@ def tls(self):
RQ_SRS_007_LDAP_Configuration_Server_EnableTLS_Options_Default("1.0")
)
def tls_enable_tls_default_yes(self):
"""Check that the default value for the `enable_tls` is set to `yes`."""
"""Check that the default value for the `enable_tls` is set to `yes`.
"""
servers = {
"openldap2": {
"host": "openldap2",
@ -171,7 +173,8 @@ def tls_enable_tls_default_yes(self):
RQ_SRS_007_LDAP_Configuration_Server_TLSRequireCert_Options_Default("1.0")
)
def tls_require_cert_default_demand(self):
"""Check that the default value for the `tls_require_cert` is set to `demand`."""
"""Check that the default value for the `tls_require_cert` is set to `demand`.
"""
servers = {
"openldap2": {
"host": "openldap2",
@ -210,7 +213,8 @@ def starttls(self):
RQ_SRS_007_LDAP_Configuration_Server_TLSCipherSuite("1.0")
)
def tls_cipher_suite(self):
"""Check that `tls_cipher_suite` parameter can be used specify allowed cipher suites."""
"""Check that `tls_cipher_suite` parameter can be used specify allowed cipher suites.
"""
servers = {
"openldap4": {
"host": "openldap4",
@ -241,7 +245,8 @@ def tls_cipher_suite(self):
])
def tls_minimum_protocol_version(self, version, exitcode, message):
"""Check that `tls_minimum_protocol_version` parameter can be used specify
to specify the minimum protocol version of SSL/TLS."""
to specify the minimum protocol version of SSL/TLS.
"""
servers = {
"openldap4": {
@ -278,6 +283,8 @@ def tls_minimum_protocol_version(self, version, exitcode, message):
@TestFeature
@Name("connection protocols")
def feature(self, node="clickhouse1"):
"""Check different LDAP connection protocols.
"""
self.context.node = self.context.cluster.node(node)
for scenario in loads(current_module(), Scenario):

3
tests/testflows/ldap/authentication/tests/multiple_servers.py
View File

@ -14,6 +14,7 @@ def scenario(self, node="clickhouse1"):
authenticate users.
"""
self.context.node = self.context.cluster.node(node)
servers = {
"openldap1": {
"host": "openldap1",
@ -35,4 +36,6 @@ def scenario(self, node="clickhouse1"):
{"server": "openldap1", "username": "user1", "password": "user1", "login": True},
{"server": "openldap2", "username": "user2", "password": "user2", "login": True}
]
with When("I add multiple LDAP servers and users that use different servers and try to login"):
login(servers, *users)

1
tests/testflows/ldap/authentication/tests/server_config.py
View File

@ -267,5 +267,6 @@ def feature(self, node="clickhouse1"):
"""Check that LDAP server configuration.
"""
self.context.node = self.context.cluster.node(node)
for scenario in loads(current_module(), Scenario):
scenario()

113
tests/testflows/ldap/external_user_directory/tests/authentications.py
View File

@ -28,7 +28,8 @@ servers = {
@TestOutline
def add_user_to_ldap_and_login(self, server, user=None, ch_user=None, login=None, exitcode=None, message=None):
"""Add user to LDAP and ClickHouse and then try to login."""
"""Add user to LDAP and ClickHouse and then try to login.
"""
self.context.ldap_node = self.context.cluster.node(server)
if ch_user is None:
@ -91,6 +92,8 @@ def parallel_login(self, server, user_count=10, timeout=200):
with Given("a group of LDAP users"):
users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)]
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
with ldap_users(*users):
tasks = []
try:
@ -124,6 +127,8 @@ def parallel_login_with_the_same_user(self, server, timeout=200):
with Given("only one LDAP user"):
users = [{"cn": f"parallel_user1", "userpassword": randomword(20)}]
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
with ldap_users(*users):
tasks = []
try:
@ -147,11 +152,11 @@ def login_after_ldap_external_user_directory_is_removed(self, server):
"""Check that ClickHouse stops authenticating LDAP users
after LDAP external user directory is removed.
"""
with When("I attempt to login after LDAP external user directory is added"):
with When("I login after LDAP external user directory is added"):
with ldap_external_user_directory(server="openldap2", roles=[], restart=True):
login_and_execute_query(username="user2", password="user2")
with When("I attempt to login after LDAP external user directory is removed"):
with And("I attempt to login after LDAP external user directory is removed"):
exitcode = 4
message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name"
login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message)
@ -318,6 +323,8 @@ def parallel_login_with_rbac_users(self, server, user_count=10, timeout=200):
users = [{"cn": f"parallel_user{i}", "userpassword": randomword(20)} for i in range(user_count)]
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
with rbac_users(*users):
tasks = []
try:
@ -336,9 +343,12 @@ def parallel_login_with_rbac_users(self, server, user_count=10, timeout=200):
RQ_SRS_009_LDAP_ExternalUserDirectory_Users_Authentication_NewUsers("1.0")
)
def login_after_user_is_added_to_ldap(self, server):
"""Check that user can login as soon as it is added to LDAP."""
"""Check that user can login as soon as it is added to LDAP.
"""
user = {"cn": "myuser", "userpassword": "myuser"}
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
with When(f"I add user to LDAP and try to login"):
add_user_to_ldap_and_login(user=user, server=server)
@ -348,10 +358,13 @@ def login_after_user_is_added_to_ldap(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_DeletedUsers("1.0")
)
def login_after_user_is_deleted_from_ldap(self, server):
"""Check that login fails after user is deleted from LDAP."""
"""Check that login fails after user is deleted from LDAP.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
try:
with Given(f"I add user to LDAP"):
user = {"cn": "myuser", "userpassword": "myuser"}
@ -378,10 +391,13 @@ def login_after_user_is_deleted_from_ldap(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_PasswordChanged("1.0")
)
def login_after_user_password_changed_in_ldap(self, server):
"""Check that login fails after user password is changed in LDAP."""
"""Check that login fails after user password is changed in LDAP.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
try:
with Given(f"I add user to LDAP"):
user = {"cn": "myuser", "userpassword": "myuser"}
@ -412,11 +428,14 @@ def login_after_user_password_changed_in_ldap(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_UsernameChanged("1.0")
)
def login_after_user_cn_changed_in_ldap(self, server):
"""Check that login fails after user cn is changed in LDAP."""
"""Check that login fails after user cn is changed in LDAP.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
new_user = None
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
try:
with Given(f"I add user to LDAP"):
user = {"cn": "myuser", "userpassword": "myuser"}
@ -443,10 +462,13 @@ def login_after_user_cn_changed_in_ldap(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_LDAPServerRestart("1.0")
)
def login_after_ldap_server_is_restarted(self, server, timeout=60):
"""Check that login succeeds after LDAP server is restarted."""
"""Check that login succeeds after LDAP server is restarted.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
try:
with Given(f"I add user to LDAP"):
user = {"cn": "myuser", "userpassword": getuid()}
@ -477,10 +499,13 @@ def login_after_ldap_server_is_restarted(self, server, timeout=60):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_ClickHouseServerRestart("1.0")
)
def login_after_clickhouse_server_is_restarted(self, server, timeout=60):
"""Check that login succeeds after ClickHouse server is restarted."""
"""Check that login succeeds after ClickHouse server is restarted.
"""
self.context.ldap_node = self.context.cluster.node(server)
user = None
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
try:
with Given(f"I add user to LDAP"):
user = {"cn": "myuser", "userpassword": getuid()}
@ -511,11 +536,14 @@ def login_after_clickhouse_server_is_restarted(self, server, timeout=60):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0")
)
def valid_username_with_valid_empty_password(self, server):
"""Check that we can't login using valid username that has empty password."""
"""Check that we can't login using valid username that has empty password.
"""
user = {"cn": "empty_password", "userpassword": ""}
exitcode = 4
message = f"DB::Exception: {user['cn']}: Authentication failed: password is incorrect or there is no user with such name"
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, exitcode=exitcode, message=message, server=server)
@TestScenario
@ -524,7 +552,8 @@ def valid_username_with_valid_empty_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Empty("1.0")
)
def valid_username_and_invalid_empty_password(self, server):
"""Check that we can't login using valid username but invalid empty password."""
"""Check that we can't login using valid username but invalid empty password.
"""
username = "user_non_empty_password"
user = {"cn": username, "userpassword": username}
login = {"password": ""}
@ -532,6 +561,8 @@ def valid_username_and_invalid_empty_password(self, server):
exitcode = 4
message = f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name"
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server)
@TestScenario
@ -539,10 +570,13 @@ def valid_username_and_invalid_empty_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Valid("1.0")
)
def valid_username_and_password(self, server):
"""Check that we can login using valid username and password."""
"""Check that we can login using valid username and password.
"""
username = "valid_username_and_password"
user = {"cn": username, "userpassword": username}
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
with When(f"I add user {username} to LDAP and try to login"):
add_user_to_ldap_and_login(user=user, server=server)
@ -552,12 +586,15 @@ def valid_username_and_password(self, server):
)
def valid_username_and_password_invalid_server(self, server=None):
"""Check that we can't login using valid username and valid
password but for a different server."""
password but for a different server.
"""
self.context.ldap_node = self.context.cluster.node("openldap1")
exitcode = 4
message = f"DB::Exception: user2: Authentication failed: password is incorrect or there is no user with such name"
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
login_and_execute_query(username="user2", password="user2", exitcode=exitcode, message=message)
@TestScenario
@ -566,10 +603,13 @@ def valid_username_and_password_invalid_server(self, server=None):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_Long("1.0"),
)
def valid_long_username_and_short_password(self, server):
"""Check that we can login using valid very long username and short password."""
"""Check that we can login using valid very long username and short password.
"""
username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890"
user = {"cn": username, "userpassword": "long_username"}
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, server=server)
@TestScenario
@ -577,7 +617,8 @@ def valid_long_username_and_short_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")
)
def invalid_long_username_and_valid_short_password(self, server):
"""Check that we can't login using slightly invalid long username but valid password."""
"""Check that we can't login using slightly invalid long username but valid password.
"""
username = "long_username_12345678901234567890123456789012345678901234567890123456789012345678901234567890"
user = {"cn": username, "userpassword": "long_username"}
login = {"username": f"{username}?"}
@ -585,6 +626,8 @@ def invalid_long_username_and_valid_short_password(self, server):
exitcode = 4
message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name"
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server)
@TestScenario
@ -593,9 +636,13 @@ def invalid_long_username_and_valid_short_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_Long("1.0")
)
def valid_short_username_and_long_password(self, server):
"""Check that we can login using valid short username with very long password."""
"""Check that we can login using valid short username with very long password.
"""
username = "long_password"
user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"}
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, server=server)
@TestScenario
@ -603,7 +650,8 @@ def valid_short_username_and_long_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")
)
def valid_short_username_and_invalid_long_password(self, server):
"""Check that we can't login using valid short username and invalid long password."""
"""Check that we can't login using valid short username and invalid long password.
"""
username = "long_password"
user = {"cn": username, "userpassword": "long_password_12345678901234567890123456789012345678901234567890123456789012345678901234567890"}
login = {"password": user["userpassword"] + "1"}
@ -611,6 +659,8 @@ def valid_short_username_and_invalid_long_password(self, server):
exitcode = 4
message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name"
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server)
@TestScenario
@ -618,7 +668,8 @@ def valid_short_username_and_invalid_long_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")
)
def valid_username_and_invalid_password(self, server):
"""Check that we can't login using valid username and invalid password."""
"""Check that we can't login using valid username and invalid password.
"""
username = "valid_username_and_invalid_password"
user = {"cn": username, "userpassword": username}
login = {"password": user["userpassword"] + "1"}
@ -626,6 +677,8 @@ def valid_username_and_invalid_password(self, server):
exitcode = 4
message=f"DB::Exception: {username}: Authentication failed: password is incorrect or there is no user with such name"
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server)
@TestScenario
@ -633,7 +686,8 @@ def valid_username_and_invalid_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Invalid("1.0")
)
def invalid_username_and_valid_password(self, server):
"""Check that we can't login using slightly invalid username but valid password."""
"""Check that we can't login using slightly invalid username but valid password.
"""
username = "invalid_username_and_valid_password"
user = {"cn": username, "userpassword": username}
login = {"username": user["cn"] + "1"}
@ -641,6 +695,8 @@ def invalid_username_and_valid_password(self, server):
exitcode = 4
message=f"DB::Exception: {login['username']}: Authentication failed: password is incorrect or there is no user with such name"
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, login=login, exitcode=exitcode, message=message, server=server)
@TestScenario
@ -649,10 +705,13 @@ def invalid_username_and_valid_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Username_UTF8("1.0")
)
def valid_utf8_username_and_ascii_password(self, server):
"""Check that we can login using valid utf-8 username with ascii password."""
"""Check that we can login using valid utf-8 username with ascii password.
"""
username = "utf8_username_Gãńdåłf_Thê_Gręât"
user = {"cn": username, "userpassword": "utf8_username"}
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, server=server)
@TestScenario
@ -661,16 +720,22 @@ def valid_utf8_username_and_ascii_password(self, server):
RQ_SRS_009_LDAP_ExternalUserDirectory_Authentication_Password_UTF8("1.0")
)
def valid_ascii_username_and_utf8_password(self, server):
"""Check that we can login using valid ascii username with utf-8 password."""
"""Check that we can login using valid ascii username with utf-8 password.
"""
username = "utf8_password"
user = {"cn": username, "userpassword": "utf8_password_Gãńdåłf_Thê_Gręât"}
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
add_user_to_ldap_and_login(user=user, server=server)
@TestScenario
def empty_username_and_empty_password(self, server=None):
"""Check that we can login using empty username and empty password as
it will use the default user and that has an empty password."""
it will use the default user and that has an empty password.
"""
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
login_and_execute_query(username="", password="")
@TestScenario
@ -698,6 +763,8 @@ def user_lookup_priority(self, server):
"ldap": {"username": "ldap", "password": "userldap"}
}
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
with ldap_users(*[{"cn": user["username"], "userpassword": user["password"]} for user in users.values()]):
with rbac_users({"cn": "local", "userpassword": "local"}):
with When("I try to login as 'default' user which is also defined in users.xml it should fail"):
@ -728,7 +795,5 @@ def feature(self, servers=None, server=None, node="clickhouse1"):
server = "openldap1"
with ldap_servers(servers):
with rbac_roles("ldap_role") as roles:
with ldap_external_user_directory(server=server, roles=roles, restart=True):
for scenario in loads(current_module(), Scenario):
Scenario(test=scenario, flags=TE)(server=server)

31
tests/testflows/ldap/external_user_directory/tests/common.py
View File

@ -70,6 +70,15 @@ def rbac_roles(*roles):
with By(f"dropping role {role}", flags=TE):
node.query(f"DROP ROLE IF EXISTS {role}")
def verify_ldap_user_exists(server, username, password):
"""Check that LDAP user is defined on the LDAP server.
"""
with By("searching LDAP database"):
ldap_node = current().context.cluster.node(server)
r = ldap_node.command(
f"ldapwhoami -H ldap://localhost -D 'cn={username},ou=users,dc=company,dc=com' -w {password}")
assert r.exitcode == 0, error()
def create_ldap_external_user_directory_config_content(server=None, roles=None, **kwargs):
"""Create LDAP external user directory configuration file content.
"""
@ -197,8 +206,26 @@ def login(servers, directory_server, *users, config=None):
@TestStep(When)
@Name("I login as {username} and execute query")
def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True, timeout=60):
def login_and_execute_query(self, username, password, exitcode=None, message=None, steps=True, timeout=60, poll=False):
if poll:
start_time = time.time()
attempt = 0
with By("repeatedly trying to login until successful or timeout"):
while True:
with When(f"attempt #{attempt}"):
r = self.context.node.query("SELECT 1", settings=[("user", username), ("password", password)],
no_checks=True, steps=False, timeout=timeout)
if r.exitcode == (0 if exitcode is None else exitcode) and (message in r.output if message is not None else True):
break
if time.time() - start_time > timeout:
fail(f"timeout {timeout} trying to login")
attempt += 1
else:
self.context.node.query("SELECT 1",
settings=[("user", username), ("password", password)],
exitcode=exitcode or 0,
exitcode=(0 if exitcode is None else exitcode),
message=message, steps=steps, timeout=timeout)

1
utils/CMakeLists.txt
View File

@ -30,6 +30,7 @@ if (NOT DEFINED ENABLE_UTILS OR ENABLE_UTILS)
add_subdirectory (checksum-for-compressed-block)
add_subdirectory (db-generator)
add_subdirectory (wal-dump)
add_subdirectory (check-mysql-binlog)
endif ()
if (ENABLE_CODE_QUALITY)

2
utils/check-mysql-binlog/CMakeLists.txt Normal file
View File

@ -0,0 +1,2 @@
add_executable(check-mysql-binlog main.cpp)
target_link_libraries(check-mysql-binlog PRIVATE dbms boost::program_options)

162
utils/check-mysql-binlog/main.cpp Normal file
View File

@ -0,0 +1,162 @@
#include <iostream>
#include <boost/program_options.hpp>
#include <IO/Operators.h>
#include <IO/ReadBufferFromFile.h>
#include <IO/ReadHelpers.h>
#include <IO/WriteHelpers.h>
#include <IO/LimitReadBuffer.h>
#include <IO/MySQLBinlogEventReadBuffer.h>
#include <IO/WriteBufferFromFileDescriptor.h>
#include <Core/MySQL/MySQLReplication.h>
static DB::MySQLReplication::BinlogEventPtr parseSingleEventBody(
DB::MySQLReplication::EventHeader & header, DB::ReadBuffer & payload,
std::shared_ptr<DB::MySQLReplication::TableMapEvent> & last_table_map_event, bool exist_checksum)
{
DB::MySQLReplication::BinlogEventPtr event;
DB::ReadBufferPtr limit_read_buffer = std::make_shared<DB::LimitReadBuffer>(payload, header.event_size - 19, false);
DB::ReadBufferPtr event_payload = limit_read_buffer;
if (exist_checksum)
event_payload = std::make_shared<DB::MySQLBinlogEventReadBuffer>(*limit_read_buffer);
switch (header.type)
{
case DB::MySQLReplication::FORMAT_DESCRIPTION_EVENT:
{
event = std::make_shared<DB::MySQLReplication::FormatDescriptionEvent>(std::move(header));
event->parseEvent(*event_payload);
break;
}
case DB::MySQLReplication::ROTATE_EVENT:
{
event = std::make_shared<DB::MySQLReplication::RotateEvent>(std::move(header));
event->parseEvent(*event_payload);
break;
}
case DB::MySQLReplication::QUERY_EVENT:
{
event = std::make_shared<DB::MySQLReplication::QueryEvent>(std::move(header));
event->parseEvent(*event_payload);
auto query = std::static_pointer_cast<DB::MySQLReplication::QueryEvent>(event);
switch (query->typ)
{
case DB::MySQLReplication::QUERY_EVENT_MULTI_TXN_FLAG:
case DB::MySQLReplication::QUERY_EVENT_XA:
{
event = std::make_shared<DB::MySQLReplication::DryRunEvent>(std::move(query->header));
break;
}
default:
break;
}
break;
}
case DB::MySQLReplication::XID_EVENT:
{
event = std::make_shared<DB::MySQLReplication::XIDEvent>(std::move(header));
event->parseEvent(*event_payload);
break;
}
case DB::MySQLReplication::TABLE_MAP_EVENT:
{
event = std::make_shared<DB::MySQLReplication::TableMapEvent>(std::move(header));
event->parseEvent(*event_payload);
last_table_map_event = std::static_pointer_cast<DB::MySQLReplication::TableMapEvent>(event);
break;
}
case DB::MySQLReplication::WRITE_ROWS_EVENT_V1:
case DB::MySQLReplication::WRITE_ROWS_EVENT_V2:
{
event = std::make_shared<DB::MySQLReplication::WriteRowsEvent>(last_table_map_event, std::move(header));
event->parseEvent(*event_payload);
break;
}
case DB::MySQLReplication::DELETE_ROWS_EVENT_V1:
case DB::MySQLReplication::DELETE_ROWS_EVENT_V2:
{
event = std::make_shared<DB::MySQLReplication::DeleteRowsEvent>(last_table_map_event, std::move(header));
event->parseEvent(*event_payload);
break;
}
case DB::MySQLReplication::UPDATE_ROWS_EVENT_V1:
case DB::MySQLReplication::UPDATE_ROWS_EVENT_V2:
{
event = std::make_shared<DB::MySQLReplication::UpdateRowsEvent>(last_table_map_event, std::move(header));
event->parseEvent(*event_payload);
break;
}
case DB::MySQLReplication::GTID_EVENT:
{
event = std::make_shared<DB::MySQLReplication::GTIDEvent>(std::move(header));
event->parseEvent(*event_payload);
break;
}
default:
{
event = std::make_shared<DB::MySQLReplication::DryRunEvent>(std::move(header));
event->parseEvent(*event_payload);
break;
}
}
return event;
}
static int checkBinLogFile(const std::string & bin_path, bool exist_checksum)
{
DB::ReadBufferFromFile in(bin_path);
DB::assertString("\xfe\x62\x69\x6e", in); /// magic number
DB::MySQLReplication::BinlogEventPtr last_event;
std::shared_ptr<DB::MySQLReplication::EventHeader> last_header;
std::shared_ptr<DB::MySQLReplication::TableMapEvent> table_map;
try
{
while (!in.eof())
{
last_header = std::make_shared<DB::MySQLReplication::EventHeader>();
last_header->parse(in);
last_event = parseSingleEventBody(*last_header, in, table_map, exist_checksum);
}
}
catch (...)
{
std::cerr << "Unable to parse MySQL binlog event. Code: " << DB::getCurrentExceptionCode() << ", Exception message: "
<< DB::getCurrentExceptionMessage(false) << std::endl << ", Previous event: " << std::endl;
last_event->dump(std::cerr);
std::cerr << std::endl << ", Event header: " << std::endl;
last_header->dump(std::cerr);
std::cerr << std::endl;
return DB::getCurrentExceptionCode();
}
std::cout << "Check passed. " << std::endl << "No exception was thrown." << std::endl << "The last binlog event: " << std::endl;
last_event->dump(std::cout);
std::cout << std::endl;
return 0;
}
int main(int argc, char ** argv)
{
boost::program_options::options_description desc("Allowed options");
desc.add_options()("help,h", "Produce help message");
desc.add_options()("disable_checksum", "Disable checksums in binlog files.");
boost::program_options::variables_map options;
boost::program_options::store(boost::program_options::parse_command_line(argc, argv, desc), options);
if (options.count("help") || argc < 2)
{
std::cout << "Usage: " << argv[0] << " mysql_binlog_file" << std::endl;
std::cout << desc << std::endl;
return 1;
}
return checkBinLogFile(argv[argc - 1], !options.count("disable_checksum"));
}