thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-20 00:30:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

Revert "less logs in WriteBufferFromS3" (#50390)

Browse Source
This commit is contained in:
Alexander Tokmakov 2023-05-31 15:26:56 +03:00 committed by GitHub
parent 3460667cac
commit 4d4112ff53
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 19 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
src/IO/WriteBufferFromS3.cpp
View File

@ -195,14 +195,18 @@ void WriteBufferFromS3::finalizeImpl()
if (request_settings.check_objects_after_upload)
{
LOG_TRACE(log, "Checking object {} exists after upload", key);
S3::checkObjectExists(*client_ptr, bucket, key, {}, request_settings, /* for_disk_s3= */ write_settings.for_object_storage, "Immediately after upload");
LOG_TRACE(log, "Checking object {} has size as expected {}", key, total_size);
size_t actual_size = S3::getObjectSize(*client_ptr, bucket, key, {}, request_settings, /* for_disk_s3= */ write_settings.for_object_storage);
if (actual_size != total_size)
throw Exception(
ErrorCodes::S3_ERROR,
"Object {} from bucket {} has unexpected size {} after upload, expected size {}, it's a bug in S3 or S3 API.",
key, bucket, actual_size, total_size);
LOG_TRACE(log, "Object {} exists after upload", key);
}
}
@ -288,6 +292,8 @@ void WriteBufferFromS3::reallocateFirstBuffer()
WriteBuffer::set(memory.data() + hidden_size, memory.size() - hidden_size);
chassert(offset() == 0);
LOG_TRACE(log, "Reallocated first buffer with size {}. {}", memory.size(), getLogDetails());
}
void WriteBufferFromS3::detachBuffer()
@ -310,6 +316,8 @@ void WriteBufferFromS3::allocateFirstBuffer()
const auto size = std::min(size_t(DBMS_DEFAULT_BUFFER_SIZE), max_first_buffer);
memory = Memory(size);
WriteBuffer::set(memory.data(), memory.size());
LOG_TRACE(log, "Allocated first buffer with size {}. {}", memory.size(), getLogDetails());
}
void WriteBufferFromS3::allocateBuffer()

11
src/IO/WriteBufferFromS3TaskTracker.cpp
View File

@ -36,6 +36,8 @@ ThreadPoolCallbackRunner<void> WriteBufferFromS3::TaskTracker::syncRunner()
void WriteBufferFromS3::TaskTracker::waitAll()
{
LOG_TEST(log, "waitAll, in queue {}", futures.size());
/// Exceptions are propagated
for (auto & future : futures)
{
@ -49,6 +51,8 @@ void WriteBufferFromS3::TaskTracker::waitAll()
void WriteBufferFromS3::TaskTracker::safeWaitAll()
{
LOG_TEST(log, "safeWaitAll, wait in queue {}", futures.size());
for (auto & future : futures)
{
if (future.valid())
@ -72,6 +76,7 @@ void WriteBufferFromS3::TaskTracker::safeWaitAll()
void WriteBufferFromS3::TaskTracker::waitIfAny()
{
LOG_TEST(log, "waitIfAny, in queue {}", futures.size());
if (futures.empty())
return;
@ -96,6 +101,8 @@ void WriteBufferFromS3::TaskTracker::waitIfAny()
watch.stop();
ProfileEvents::increment(ProfileEvents::WriteBufferFromS3WaitInflightLimitMicroseconds, watch.elapsedMicroseconds());
LOG_TEST(log, "waitIfAny ended, in queue {}", futures.size());
}
void WriteBufferFromS3::TaskTracker::add(Callback && func)
@ -140,6 +147,8 @@ void WriteBufferFromS3::TaskTracker::waitTilInflightShrink()
if (!max_tasks_inflight)
return;
LOG_TEST(log, "waitTilInflightShrink, in queue {}", futures.size());
Stopwatch watch;
/// Alternative approach is to wait until at least futures.size() - max_tasks_inflight element are finished
@ -162,6 +171,8 @@ void WriteBufferFromS3::TaskTracker::waitTilInflightShrink()
watch.stop();
ProfileEvents::increment(ProfileEvents::WriteBufferFromS3WaitInflightLimitMicroseconds, watch.elapsedMicroseconds());
LOG_TEST(log, "waitTilInflightShrink ended, in queue {}", futures.size());
}
bool WriteBufferFromS3::TaskTracker::isAsync() const