thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16727 from azat/distribution_queue-masking

Browse Source 
Mask password in data_path in the system.distribution_queue
This commit is contained in:
alexey-milovidov 2020-11-07 13:40:41 +03:00 committed by GitHub
commit 8dbd04a229
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 136 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
programs/server/config.d/test_cluster_with_incorrect_pw.xml Symbolic link
View File

@ -0,0 +1 @@
../../../tests/config/config.d/test_cluster_with_incorrect_pw.xml

73
src/Storages/System/StorageSystemDistributionQueue.cpp
View File

@ -10,6 +10,77 @@
#include <Common/typeid_cast.h>
#include <Databases/IDatabase.h>
namespace DB
{
namespace ErrorCodes
{
extern const int LOGICAL_ERROR;
}
}
namespace
{
using namespace DB;
/// Drop "password" from the path.
///
/// In case of use_compact_format_in_distributed_parts_names=0 the path format is:
///
/// user[:password]@host:port#default_database format
///
/// And password should be masked out.
///
/// See:
/// - Cluster::Address::fromFullString()
/// - Cluster::Address::toFullString()
std::string maskDataPath(const std::string & path)
{
std::string masked_path = path;
if (!masked_path.ends_with('/'))
throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid path format");
masked_path.pop_back();
size_t node_pos = masked_path.rfind('/');
/// Loop through each node, that separated with a comma
while (node_pos != std::string::npos)
{
++node_pos;
size_t user_pw_end = masked_path.find('@', node_pos);
if (user_pw_end == std::string::npos)
{
/// Likey new format (use_compact_format_in_distributed_parts_names=1)
return path;
}
size_t pw_start = masked_path.find(':', node_pos);
if (pw_start > user_pw_end)
{
/// No password in path
return path;
}
++pw_start;
size_t pw_length = user_pw_end - pw_start;
/// Replace with a single '*' to hide even the password length.
masked_path.replace(pw_start, pw_length, 1, '*');
/// "," cannot be in the node specification since it will be encoded in hex.
node_pos = masked_path.find(',', node_pos);
}
masked_path.push_back('/');
return masked_path;
}
}
namespace DB
{
@ -103,7 +174,7 @@ void StorageSystemDistributionQueue::fillData(MutableColumns & res_columns, cons
size_t col_num = 0;
res_columns[col_num++]->insert(database);
res_columns[col_num++]->insert(table);
res_columns[col_num++]->insert(status.path);
res_columns[col_num++]->insert(maskDataPath(status.path));
res_columns[col_num++]->insert(status.is_blocked);
res_columns[col_num++]->insert(status.error_count);
res_columns[col_num++]->insert(status.files_count);

21
tests/config/config.d/test_cluster_with_incorrect_pw.xml Normal file
View File

@ -0,0 +1,21 @@
<yandex>
<remote_servers>
<test_cluster_with_incorrect_pw>
<shard>
<internal_replication>true</internal_replication>
<replica>
<host>127.0.0.1</host>
<port>9000</port>
<!-- password is incorrect -->
<password>foo</password>
</replica>
<replica>
<host>127.0.0.2</host>
<port>9000</port>
<!-- password is incorrect -->
<password>foo</password>
</replica>
</shard>
</test_cluster_with_incorrect_pw>
</remote_servers>
</yandex>

1
tests/config/install.sh
View File

@ -27,6 +27,7 @@ ln -sf $SRC_PATH/config.d/secure_ports.xml $DEST_SERVER_PATH/config.d/
ln -sf $SRC_PATH/config.d/clusters.xml $DEST_SERVER_PATH/config.d/
ln -sf $SRC_PATH/config.d/graphite.xml $DEST_SERVER_PATH/config.d/
ln -sf $SRC_PATH/config.d/database_atomic.xml $DEST_SERVER_PATH/config.d/
ln -sf $SRC_PATH/config.d/test_cluster_with_incorrect_pw.xml $DEST_SERVER_PATH/config.d/
ln -sf $SRC_PATH/users.d/log_queries.xml $DEST_SERVER_PATH/users.d/
ln -sf $SRC_PATH/users.d/readonly.xml $DEST_SERVER_PATH/users.d/
ln -sf $SRC_PATH/users.d/access_management.xml $DEST_SERVER_PATH/users.d/

4
tests/queries/0_stateless/01555_system_distribution_queue_mask.reference Normal file
View File

@ -0,0 +1,4 @@
masked
3,"default:*@127%2E0%2E0%2E1:9000,default:*@127%2E0%2E0%2E2:9000"
no masking
1,"default@localhost:9000"

36
tests/queries/0_stateless/01555_system_distribution_queue_mask.sql Normal file
View File

@ -0,0 +1,36 @@
-- force data path with the user/pass in it
set use_compact_format_in_distributed_parts_names=0;
-- use async send even for localhost
set prefer_localhost_replica=0;
drop table if exists dist_01555;
drop table if exists data_01555;
create table data_01555 (key Int) Engine=Null();
--
-- masked
--
SELECT 'masked';
create table dist_01555 (key Int) Engine=Distributed(test_cluster_with_incorrect_pw, currentDatabase(), data_01555, key);
insert into dist_01555 values (1)(2);
-- since test_cluster_with_incorrect_pw contains incorrect password ignore error
system flush distributed dist_01555; -- { serverError 516; }
select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1') from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV;
drop table dist_01555;
--
-- no masking
--
SELECT 'no masking';
create table dist_01555 (key Int) Engine=Distributed(test_shard_localhost, currentDatabase(), data_01555, key);
insert into dist_01555 values (1)(2);
-- since test_cluster_with_incorrect_pw contains incorrect password ignore error
system flush distributed dist_01555;
select length(splitByChar('*', data_path)), replaceRegexpOne(data_path, '^.*/([^/]*)/' , '\\1') from system.distribution_queue where database = currentDatabase() and table = 'dist_01555' format CSV;
-- cleanup
drop table dist_01555;
drop table data_01555;

1
tests/queries/0_stateless/arcadia_skip_list.txt
View File

@ -163,4 +163,5 @@
01547_query_log_current_database
01548_query_log_query_execution_ms
01552_dict_fixedstring
01555_system_distribution_queue_mask
01557_max_parallel_replicas_no_sample.sql