thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-23 08:02:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Load base backups lazily (if a backup is not needed it won't be loaded).

Browse Source
This commit is contained in:
Vitaly Baranov 2023-11-09 14:21:42 +01:00
parent bc25839be8
commit 1711bed63e
6 changed files with 180 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

107
src/Backups/BackupImpl.cpp
View File

@ -3,6 +3,7 @@
#include <Backups/BackupFileInfo.h>
#include <Backups/BackupIO.h>
#include <Backups/IBackupEntry.h>
#include <Common/ProfileEvents.h>
#include <Common/StringUtils/StringUtils.h>
#include <base/hex.h>
#include <Common/logger_useful.h>
@ -24,6 +25,14 @@
#include <Poco/DOM/DOMParser.h>
namespace ProfileEvents
{
extern const Event BackupsOpenedForRead;
extern const Event BackupsOpenedForWrite;
extern const Event BackupReadMetadataMicroseconds;
extern const Event BackupWriteMetadataMicroseconds;
}
namespace DB
{
namespace ErrorCodes
@ -89,12 +98,14 @@ BackupImpl::BackupImpl(
, archive_params(archive_params_)
, open_mode(OpenMode::READ)
, reader(std::move(reader_))
, context(context_)
, is_internal_backup(false)
, version(INITIAL_BACKUP_VERSION)
, base_backup_info(base_backup_info_)
, use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
, log(&Poco::Logger::get("BackupImpl"))
{
open(context_);
open();
}
@ -115,6 +126,7 @@ BackupImpl::BackupImpl(
, archive_params(archive_params_)
, open_mode(OpenMode::WRITE)
, writer(std::move(writer_))
, context(context_)
, is_internal_backup(is_internal_backup_)
, coordination(coordination_)
, uuid(backup_uuid_)
@ -124,7 +136,7 @@ BackupImpl::BackupImpl(
, use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
, log(&Poco::Logger::get("BackupImpl"))
{
open(context_);
open();
}
@ -140,9 +152,11 @@ BackupImpl::~BackupImpl()
}
}
void BackupImpl::open(const ContextPtr & context)
void BackupImpl::open()
{
std::lock_guard lock{mutex};
LOG_INFO(log, "{} backup: {}", ((open_mode == OpenMode::WRITE) ? "Writing" : "Reading"), backup_name_for_logging);
ProfileEvents::increment((open_mode == OpenMode::WRITE) ? ProfileEvents::BackupsOpenedForWrite : ProfileEvents::BackupsOpenedForRead);
if (open_mode == OpenMode::WRITE)
{
@ -166,35 +180,8 @@ void BackupImpl::open(const ContextPtr & context)
if (open_mode == OpenMode::READ)
readBackupMetadata();
if (base_backup_info)
{
if (use_same_s3_credentials_for_base_backup)
backup_info.copyS3CredentialsTo(*base_backup_info);
BackupFactory::CreateParams params;
params.backup_info = *base_backup_info;
params.open_mode = OpenMode::READ;
params.context = context;
/// use_same_s3_credentials_for_base_backup should be inherited for base backups
params.use_same_s3_credentials_for_base_backup = use_same_s3_credentials_for_base_backup;
base_backup = BackupFactory::instance().createBackup(params);
if (open_mode == OpenMode::WRITE)
{
base_backup_uuid = base_backup->getUUID();
}
else if (base_backup_uuid != base_backup->getUUID())
{
throw Exception(
ErrorCodes::WRONG_BASE_BACKUP,
"Backup {}: The base backup {} has different UUID ({} != {})",
backup_name_for_logging,
base_backup->getNameForLogging(),
toString(base_backup->getUUID()),
(base_backup_uuid ? toString(*base_backup_uuid) : ""));
}
}
if ((open_mode == OpenMode::WRITE) && base_backup_info)
base_backup_uuid = getBaseBackupUnlocked()->getUUID();
}
void BackupImpl::close()
@ -239,6 +226,42 @@ void BackupImpl::closeArchive()
archive_writer.reset();
}
std::shared_ptr<const IBackup> BackupImpl::getBaseBackup() const
{
std::lock_guard lock{mutex};
return getBaseBackupUnlocked();
}
std::shared_ptr<const IBackup> BackupImpl::getBaseBackupUnlocked() const
{
if (!base_backup && base_backup_info)
{
if (use_same_s3_credentials_for_base_backup)
backup_info.copyS3CredentialsTo(*base_backup_info);
BackupFactory::CreateParams params;
params.backup_info = *base_backup_info;
params.open_mode = OpenMode::READ;
params.context = context;
/// use_same_s3_credentials_for_base_backup should be inherited for base backups
params.use_same_s3_credentials_for_base_backup = use_same_s3_credentials_for_base_backup;
base_backup = BackupFactory::instance().createBackup(params);
if ((open_mode == OpenMode::READ) && (base_backup_uuid != base_backup->getUUID()))
{
throw Exception(
ErrorCodes::WRONG_BASE_BACKUP,
"Backup {}: The base backup {} has different UUID ({} != {})",
backup_name_for_logging,
base_backup->getNameForLogging(),
toString(base_backup->getUUID()),
(base_backup_uuid ? toString(*base_backup_uuid) : ""));
}
}
return base_backup;
}
size_t BackupImpl::getNumFiles() const
{
std::lock_guard lock{mutex};
@ -289,8 +312,10 @@ UInt64 BackupImpl::getNumReadBytes() const
void BackupImpl::writeBackupMetadata()
{
assert(!is_internal_backup);
LOG_TRACE(log, "Backup {}: Writing metadata", backup_name_for_logging);
auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::BackupWriteMetadataMicroseconds);
assert(!is_internal_backup);
checkLockFile(true);
std::unique_ptr<WriteBuffer> out;
@ -374,11 +399,16 @@ void BackupImpl::writeBackupMetadata()
out->finalize();
uncompressed_size = size_of_entries + out->count();
LOG_TRACE(log, "Backup {}: Metadata was written", backup_name_for_logging);
}
void BackupImpl::readBackupMetadata()
{
LOG_TRACE(log, "Backup {}: Reading metadata", backup_name_for_logging);
auto timer = DB::CurrentThread::getProfileEvents().timer(ProfileEvents::BackupReadMetadataMicroseconds);
using namespace XMLUtils;
std::unique_ptr<ReadBuffer> in;
@ -482,6 +512,8 @@ void BackupImpl::readBackupMetadata()
compressed_size = uncompressed_size;
if (!use_archive)
setCompressedSize();
LOG_TRACE(log, "Backup {}: Metadata was read", backup_name_for_logging);
}
void BackupImpl::checkBackupDoesntExist() const
@ -705,7 +737,8 @@ std::unique_ptr<SeekableReadBuffer> BackupImpl::readFileImpl(const SizeAndChecks
if (info.base_size)
{
/// Make `base_read_buffer` if there is data for this backup entry in the base backup.
if (!base_backup)
auto base = getBaseBackup();
if (!base)
{
throw Exception(
ErrorCodes::NO_BASE_BACKUP,
@ -713,7 +746,7 @@ std::unique_ptr<SeekableReadBuffer> BackupImpl::readFileImpl(const SizeAndChecks
backup_name_for_logging, formatSizeAndChecksum(size_and_checksum));
}
if (!base_backup->fileExists(std::pair(info.base_size, info.base_checksum)))
if (!base->fileExists(std::pair(info.base_size, info.base_checksum)))
{
throw Exception(
ErrorCodes::WRONG_BASE_BACKUP,
@ -721,7 +754,7 @@ std::unique_ptr<SeekableReadBuffer> BackupImpl::readFileImpl(const SizeAndChecks
backup_name_for_logging, formatSizeAndChecksum(size_and_checksum));
}
base_read_buffer = base_backup->readFile(std::pair{info.base_size, info.base_checksum});
base_read_buffer = base->readFile(std::pair{info.base_size, info.base_checksum});
}
{
@ -809,7 +842,7 @@ size_t BackupImpl::copyFileToDisk(const SizeAndChecksum & size_and_checksum,
else if (info.size && (info.size == info.base_size))
{
/// Data comes completely from the base backup (nothing comes from this backup).
base_backup->copyFileToDisk(std::pair{info.base_size, info.base_checksum}, destination_disk, destination_path, write_mode);
getBaseBackup()->copyFileToDisk(std::pair{info.base_size, info.base_checksum}, destination_disk, destination_path, write_mode);
file_copied = true;
}

12
src/Backups/BackupImpl.h
View File

@ -60,7 +60,7 @@ public:
OpenMode getOpenMode() const override { return open_mode; }
time_t getTimestamp() const override { return timestamp; }
UUID getUUID() const override { return *uuid; }
BackupPtr getBaseBackup() const override { return base_backup; }
BackupPtr getBaseBackup() const override;
size_t getNumFiles() const override;
UInt64 getTotalSize() const override;
size_t getNumEntries() const override;
@ -85,7 +85,7 @@ public:
bool supportsWritingInMultipleThreads() const override { return !use_archive; }
private:
void open(const ContextPtr & context);
void open();
void close();
void openArchive();
@ -95,6 +95,9 @@ private:
void writeBackupMetadata() TSA_REQUIRES(mutex);
void readBackupMetadata() TSA_REQUIRES(mutex);
/// Returns the base backup or null if there is no base backup.
std::shared_ptr<const IBackup> getBaseBackupUnlocked() const TSA_REQUIRES(mutex);
/// Checks that a new backup doesn't exist yet.
void checkBackupDoesntExist() const;
@ -118,6 +121,7 @@ private:
const OpenMode open_mode;
std::shared_ptr<IBackupWriter> writer;
std::shared_ptr<IBackupReader> reader;
const ContextPtr context;
const bool is_internal_backup;
std::shared_ptr<IBackupCoordination> coordination;
@ -138,8 +142,8 @@ private:
mutable size_t num_read_files = 0;
mutable UInt64 num_read_bytes = 0;
int version;
std::optional<BackupInfo> base_backup_info;
std::shared_ptr<const IBackup> base_backup;
mutable std::optional<BackupInfo> base_backup_info;
mutable std::shared_ptr<const IBackup> base_backup;
std::optional<UUID> base_backup_uuid;
std::shared_ptr<IArchiveReader> archive_reader;
std::shared_ptr<IArchiveWriter> archive_writer;

2
src/Backups/IBackup.h
View File

@ -43,7 +43,7 @@ public:
/// Returns UUID of the backup.
virtual UUID getUUID() const = 0;
/// Returns the base backup (can be null).
/// Returns the base backup or null if there is no base backup.
virtual std::shared_ptr<const IBackup> getBaseBackup() const = 0;
/// Returns the number of files stored in the backup. Compare with getNumEntries().

4
src/Common/ProfileEvents.cpp
View File

@ -546,6 +546,10 @@ The server successfully detected this situation and will download merged part fr
M(IOUringCQEsCompleted, "Total number of successfully completed io_uring CQEs") \
M(IOUringCQEsFailed, "Total number of completed io_uring CQEs with failures") \
\
M(BackupsOpenedForRead, "Number of backups opened for reading") \
M(BackupsOpenedForWrite, "Number of backups opened for writing") \
M(BackupReadMetadataMicroseconds, "Time spent reading backup metadata from .backup file") \
M(BackupWriteMetadataMicroseconds, "Time spent writing backup metadata to .backup file") \
M(BackupEntriesCollectorMicroseconds, "Time spent making backup entries") \
M(BackupEntriesCollectorForTablesDataMicroseconds, "Time spent making backup entries for tables data") \
M(BackupEntriesCollectorRunPostTasksMicroseconds, "Time spent running post tasks after making backup entries") \

12
tests/queries/0_stateless/02915_lazy_loading_of_base_backups.reference Normal file
View File

@ -0,0 +1,12 @@
BACKUP_CREATED
BACKUP_CREATED
BACKUP_CREATED
RESTORED
RESTORED
RESTORED
a 0 1
b 1 1
c 1 1
r1 3 0
r2 2 0
r3 1 0

85
tests/queries/0_stateless/02915_lazy_loading_of_base_backups.sh Executable file
View File

@ -0,0 +1,85 @@
#!/usr/bin/env bash
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
a_backup_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_a
a_backup="Disk('backups', '$a_backup_id')"
b_backup_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_b
b_backup="Disk('backups', '$b_backup_id')"
c_backup_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_c
c_backup="Disk('backups', '$c_backup_id')"
${CLICKHOUSE_CLIENT} -nm --query "
DROP TABLE IF EXISTS tbl1;
DROP TABLE IF EXISTS tbl2;
DROP TABLE IF EXISTS tbl3;
"
${CLICKHOUSE_CLIENT} -nm --query "
CREATE TABLE tbl1 (a Int32) ENGINE = MergeTree() ORDER BY tuple();
"
# The following BACKUP command must write backup 'a'.
${CLICKHOUSE_CLIENT} -nm --query "
BACKUP DATABASE ${CLICKHOUSE_DATABASE} TO $a_backup SETTINGS id='$a_backup_id';
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
CREATE TABLE tbl2 (a Int32) ENGINE = MergeTree() ORDER BY tuple();
"
# The following BACKUP command must read backup 'a' and write backup 'b'.
${CLICKHOUSE_CLIENT} -nm --query "
BACKUP DATABASE ${CLICKHOUSE_DATABASE} TO $b_backup SETTINGS id='$b_backup_id', base_backup=$a_backup;
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
CREATE TABLE tbl3 (a Int32) ENGINE = MergeTree() ORDER BY tuple();
"
# The following BACKUP command must read only backup 'b' (and not 'a') and write backup 'c'.
${CLICKHOUSE_CLIENT} -nm --query "
BACKUP DATABASE ${CLICKHOUSE_DATABASE} TO $c_backup SETTINGS id='$c_backup_id', base_backup=$b_backup;
" | grep -o "BACKUP_CREATED"
${CLICKHOUSE_CLIENT} -nm --query "
DROP TABLE tbl1;
DROP TABLE tbl2;
DROP TABLE tbl3;
"
r1_restore_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_r1
r2_restore_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_r2
r3_restore_id=${CLICKHOUSE_TEST_UNIQUE_NAME}_r3
# The following RESTORE command must read all 3 backups 'a', 'b', c' because the table 'tbl1' was in the first backup.
${CLICKHOUSE_CLIENT} -nm --query "
RESTORE TABLE ${CLICKHOUSE_DATABASE}.tbl1 FROM $c_backup SETTINGS id='$r1_restore_id';
" | grep -o "RESTORED"
# The following RESTORE command must read only 2 backups 'b', c' (and not 'a') because the table 'tbl2' was in the second backup.
${CLICKHOUSE_CLIENT} -nm --query "
RESTORE TABLE ${CLICKHOUSE_DATABASE}.tbl2 FROM $c_backup SETTINGS id='$r2_restore_id';
" | grep -o "RESTORED"
# The following RESTORE command must read only 1 backup 'c' (and not 'a' or 'b') because the table 'tbl3' was in the third backup.
${CLICKHOUSE_CLIENT} -nm --query "
RESTORE TABLE ${CLICKHOUSE_DATABASE}.tbl3 FROM $c_backup SETTINGS id='$r3_restore_id';
" | grep -o "RESTORED"
all_ids="['$a_backup_id', '$b_backup_id', '$c_backup_id', '$r1_restore_id', '$r2_restore_id', '$r3_restore_id']"
id_prefix_len=`expr "${CLICKHOUSE_TEST_UNIQUE_NAME}_" : '.*'`
${CLICKHOUSE_CLIENT} -nm --query "
SELECT substr(id, 1 + $id_prefix_len) as short_id, ProfileEvents['BackupsOpenedForRead'], ProfileEvents['BackupsOpenedForWrite'] FROM system.backups WHERE id IN ${all_ids} ORDER BY short_id
"
${CLICKHOUSE_CLIENT} -nm --query "
DROP TABLE tbl1;
DROP TABLE tbl2;
DROP TABLE tbl3;
"