thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #53750 from ClickHouse/Add_retries_for_azure_retryable_errors

Browse Source 
Add retries for azure retryable errors
This commit is contained in:
Alexander Tokmakov 2023-08-24 20:26:42 +03:00 committed by GitHub
commit 75d32bfe77
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 31 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
src/Storages/MergeTree/MergeTreeData.cpp
View File

@ -103,6 +103,10 @@
#include <Poco/Logger.h>
#include <Poco/Net/NetException.h>
#if USE_AZURE_BLOB_STORAGE
#include <azure/core/http/http.hpp>
#endif
template <>
struct fmt::formatter<DB::DataPartPtr> : fmt::formatter<std::string>
{
@ -1248,6 +1252,12 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPart(
{
throw;
}
#if USE_AZURE_BLOB_STORAGE
catch (const Azure::Core::Http::TransportException &)
{
throw;
}
#endif
catch (...)
{
mark_broken();
@ -1396,6 +1406,18 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPartWithRetries(
size_t max_backoff_ms,
size_t max_tries)
{
auto handle_exception = [&, this](String exception_message, size_t try_no)
{
if (try_no + 1 == max_tries)
throw;
LOG_DEBUG(log, "Failed to load data part {} at try {} with retryable error: {}. Will retry in {} ms",
part_name, try_no, exception_message, initial_backoff_ms);
std::this_thread::sleep_for(std::chrono::milliseconds(initial_backoff_ms));
initial_backoff_ms = std::min(initial_backoff_ms * 2, max_backoff_ms);
};
for (size_t try_no = 0; try_no < max_tries; ++try_no)
{
try
@ -1404,15 +1426,17 @@ MergeTreeData::LoadPartResult MergeTreeData::loadDataPartWithRetries(
}
catch (const Exception & e)
{
if (!isRetryableException(e) || try_no + 1 == max_tries)
if (isRetryableException(e))
handle_exception(e.message(),try_no);
else
throw;
LOG_DEBUG(log, "Failed to load data part {} at try {} with retryable error: {}. Will retry in {} ms",
part_name, try_no, e.message(), initial_backoff_ms);
std::this_thread::sleep_for(std::chrono::milliseconds(initial_backoff_ms));
initial_backoff_ms = std::min(initial_backoff_ms * 2, max_backoff_ms);
}
#if USE_AZURE_BLOB_STORAGE
catch (const Azure::Core::Http::TransportException & e)
{
handle_exception(e.Message,try_no);
}
#endif
}
UNREACHABLE();
}

1
src/Storages/MergeTree/checkDataPart.cpp
View File

@ -16,7 +16,6 @@
#include <IO/S3Common.h>
#include <Common/CurrentMetrics.h>
namespace CurrentMetrics
{
extern const Metric ReplicatedChecks;