thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 23:21:59 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into retry-more-error-codes-from-s3

Browse Source
This commit is contained in:
Alexey Milovidov 2024-10-28 06:35:15 +01:00
commit 29df81ff12
6 changed files with 51 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
programs/server/Server.cpp
View File

@ -2271,10 +2271,19 @@ try
if (has_zookeeper && global_context->getMacros()->getMacroMap().contains("replica"))
{
auto zookeeper = global_context->getZooKeeper();
String stop_flag_path = "/clickhouse/stop_replicated_ddl_queries/{replica}";
stop_flag_path = global_context->getMacros()->expand(stop_flag_path);
found_stop_flag = zookeeper->exists(stop_flag_path);
try
{
auto zookeeper = global_context->getZooKeeper();
String stop_flag_path = "/clickhouse/stop_replicated_ddl_queries/{replica}";
stop_flag_path = global_context->getMacros()->expand(stop_flag_path);
found_stop_flag = zookeeper->exists(stop_flag_path);
}
catch (const Coordination::Exception & e)
{
if (e.code != Coordination::Error::ZCONNECTIONLOSS)
throw;
tryLogCurrentException(log);
}
}
if (found_stop_flag)

4
src/IO/S3/AWSLogger.cpp
View File

@ -17,7 +17,7 @@ const char * S3_LOGGER_TAG_NAMES[][2] = {
const std::pair<DB::LogsLevel, Poco::Message::Priority> & convertLogLevel(Aws::Utils::Logging::LogLevel log_level)
{
/// We map levels to our own logger 1 to 1 except WARN+ levels. In most cases we failover such errors with retries
/// We map levels to our own logger 1 to 1 except INFO+ levels. In most cases we fail over such errors with retries
/// and don't want to see them as Errors in our logs.
static const std::unordered_map<Aws::Utils::Logging::LogLevel, std::pair<DB::LogsLevel, Poco::Message::Priority>> mapping =
{
@ -25,7 +25,7 @@ const std::pair<DB::LogsLevel, Poco::Message::Priority> & convertLogLevel(Aws::U
{Aws::Utils::Logging::LogLevel::Fatal, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
{Aws::Utils::Logging::LogLevel::Error, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
{Aws::Utils::Logging::LogLevel::Warn, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
{Aws::Utils::Logging::LogLevel::Info, {DB::LogsLevel::information, Poco::Message::PRIO_INFORMATION}},
{Aws::Utils::Logging::LogLevel::Info, {DB::LogsLevel::debug, Poco::Message::PRIO_DEBUG}},
{Aws::Utils::Logging::LogLevel::Debug, {DB::LogsLevel::debug, Poco::Message::PRIO_TEST}},
{Aws::Utils::Logging::LogLevel::Trace, {DB::LogsLevel::trace, Poco::Message::PRIO_TEST}},
};

6
src/IO/S3/deleteFileFromS3.cpp
View File

@ -56,7 +56,7 @@ void deleteFileFromS3(
if (outcome.IsSuccess())
{
LOG_INFO(log, "Object with path {} was removed from S3", key);
LOG_DEBUG(log, "Object with path {} was removed from S3", key);
}
else if (if_exists && S3::isNotFoundError(outcome.GetError().GetErrorType()))
{
@ -173,7 +173,7 @@ void deleteFilesFromS3(
if (errors.empty())
{
/// All the objects were removed.
LOG_INFO(log, "Objects with paths [{}] were removed from S3", comma_separated_keys);
LOG_DEBUG(log, "Objects with paths [{}] were removed from S3", comma_separated_keys);
}
else
{
@ -210,7 +210,7 @@ void deleteFilesFromS3(
removed_keys_comma_separated += ", ";
removed_keys_comma_separated += key;
}
LOG_INFO(log, "Objects with paths [{}] were removed from S3", removed_keys_comma_separated);
LOG_DEBUG(log, "Objects with paths [{}] were removed from S3", removed_keys_comma_separated);
}
if (!not_found_keys.empty())

10
src/IO/WriteBufferFromHTTP.cpp
View File

@ -24,17 +24,17 @@ WriteBufferFromHTTP::WriteBufferFromHTTP(
request.setHost(uri.getHost());
request.setChunkedTransferEncoding(true);
if (!content_type.empty())
{
request.set("Content-Type", content_type);
}
if (!content_encoding.empty())
request.set("Content-Encoding", content_encoding);
for (const auto & header: additional_headers)
request.add(header.name, header.value);
if (!content_type.empty() && !request.has("Content-Type"))
{
request.set("Content-Type", content_type);
}
LOG_TRACE((getLogger("WriteBufferToHTTP")), "Sending request to {}", uri.toString());
ostr = &session->sendRequest(request);

2
tests/integration/test_grant_and_revoke/test_with_table_engine_grant.py
View File

@ -192,7 +192,7 @@ def test_grant_all_on_table():
instance.query("GRANT ALL ON test.table TO B", user="A")
assert (
instance.query("SHOW GRANTS FOR B")
== "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, UNDROP TABLE, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM PULLING REPLICATION LOG, SYSTEM CLEANUP, SYSTEM VIEWS, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM VIRTUAL PARTS UPDATE, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM FLUSH DISTRIBUTED, SYSTEM UNLOAD PRIMARY KEY, dictGet ON test.`table` TO B\n"
== "GRANT SHOW TABLES, SHOW COLUMNS, SHOW DICTIONARIES, SELECT, INSERT, ALTER TABLE, ALTER VIEW, CREATE TABLE, CREATE VIEW, CREATE DICTIONARY, DROP TABLE, DROP VIEW, DROP DICTIONARY, UNDROP TABLE, TRUNCATE, OPTIMIZE, BACKUP, CREATE ROW POLICY, ALTER ROW POLICY, DROP ROW POLICY, SHOW ROW POLICIES, SYSTEM MERGES, SYSTEM TTL MERGES, SYSTEM FETCHES, SYSTEM MOVES, SYSTEM PULLING REPLICATION LOG, SYSTEM CLEANUP, SYSTEM VIEWS, SYSTEM SENDS, SYSTEM REPLICATION QUEUES, SYSTEM VIRTUAL PARTS UPDATE, SYSTEM REDUCE BLOCKING PARTS, SYSTEM DROP REPLICA, SYSTEM SYNC REPLICA, SYSTEM RESTART REPLICA, SYSTEM RESTORE REPLICA, SYSTEM WAIT LOADING PARTS, SYSTEM FLUSH DISTRIBUTED, SYSTEM UNLOAD PRIMARY KEY, dictGet ON test.`table` TO B\n"
)
instance.query("REVOKE ALL ON test.table FROM B", user="A")
assert instance.query("SHOW GRANTS FOR B") == ""

27
tests/integration/test_storage_url_http_headers/test.py
View File

@ -114,3 +114,30 @@ def test_storage_url_redirected_headers(started_cluster):
assert "Host: 127.0.0.1" not in result
assert "Host: localhost" in result
def test_with_override_content_type_url_http_headers(started_cluster):
query = "INSERT INTO TABLE FUNCTION url('http://localhost:8000/', JSONEachRow, 'x UInt8') SELECT 1"
server.query(query)
result = server.exec_in_container(
["cat", http_headers_echo_server.RESULT_PATH], user="root"
)
print(result)
assert "Content-Type: application/x-ndjson; charset=UTF-8" in result
query = "INSERT INTO TABLE FUNCTION url('http://localhost:8000/', JSONEachRow, 'x UInt8', headers('Content-Type' = 'upyachka')) SELECT 1"
server.query(query)
result = server.exec_in_container(
["cat", http_headers_echo_server.RESULT_PATH], user="root"
)
print(result)
assert "Content-Type: application/x-ndjson; charset=UTF-8" not in result
assert "Content-Type: upyachka" in result