thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-22 15:42:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #66214 from sguerrini97/use_same_password_for_base_backup

Browse Source 
Allow to create incremental password protected backups
This commit is contained in:
János Benjamin Antal 2024-08-05 11:18:21 +00:00 committed by GitHub
commit 8fece64f33
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 111 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
docs/en/operations/backup.md
View File

@ -85,6 +85,7 @@ The BACKUP and RESTORE statements take a list of DATABASE and TABLE names, a des
- `password` for the file on disk
- `base_backup`: the destination of the previous backup of this source. For example, `Disk('backups', '1.zip')`
- `use_same_s3_credentials_for_base_backup`: whether base backup to S3 should inherit credentials from the query. Only works with `S3`.
- `use_same_password_for_base_backup`: whether base backup archive should inherit the password from the query.
- `structure_only`: if enabled, allows to only backup or restore the CREATE statements without the data of tables
- `storage_policy`: storage policy for the tables being restored. See [Using Multiple Block Devices for Data Storage](../engines/table-engines/mergetree-family/mergetree.md#table_engine-mergetree-multiple-volumes). This setting is only applicable to the `RESTORE` command. The specified storage policy applies only to tables with an engine from the `MergeTree` family.
- `s3_storage_class`: the storage class used for S3 backup. For example, `STANDARD`

1
src/Backups/BackupFactory.h
View File

@ -41,6 +41,7 @@ public:
bool allow_s3_native_copy = true;
bool allow_azure_native_copy = true;
bool use_same_s3_credentials_for_base_backup = false;
bool use_same_password_for_base_backup = false;
bool azure_attempt_to_create_container = true;
ReadSettings read_settings;
WriteSettings write_settings;

13
src/Backups/BackupImpl.cpp
View File

@ -92,7 +92,8 @@ BackupImpl::BackupImpl(
std::shared_ptr<IBackupReader> reader_,
const ContextPtr & context_,
bool is_internal_backup_,
bool use_same_s3_credentials_for_base_backup_)
bool use_same_s3_credentials_for_base_backup_,
bool use_same_password_for_base_backup_)
: backup_info(backup_info_)
, backup_name_for_logging(backup_info.toStringForLogging())
, use_archive(!archive_params_.archive_name.empty())
@ -104,6 +105,7 @@ BackupImpl::BackupImpl(
, version(INITIAL_BACKUP_VERSION)
, base_backup_info(base_backup_info_)
, use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
, use_same_password_for_base_backup(use_same_password_for_base_backup_)
, log(getLogger("BackupImpl"))
{
open();
@ -120,7 +122,8 @@ BackupImpl::BackupImpl(
const std::shared_ptr<IBackupCoordination> & coordination_,
const std::optional<UUID> & backup_uuid_,
bool deduplicate_files_,
bool use_same_s3_credentials_for_base_backup_)
bool use_same_s3_credentials_for_base_backup_,
bool use_same_password_for_base_backup_)
: backup_info(backup_info_)
, backup_name_for_logging(backup_info.toStringForLogging())
, use_archive(!archive_params_.archive_name.empty())
@ -135,6 +138,7 @@ BackupImpl::BackupImpl(
, base_backup_info(base_backup_info_)
, deduplicate_files(deduplicate_files_)
, use_same_s3_credentials_for_base_backup(use_same_s3_credentials_for_base_backup_)
, use_same_password_for_base_backup(use_same_password_for_base_backup_)
, log(getLogger("BackupImpl"))
{
open();
@ -258,6 +262,11 @@ std::shared_ptr<const IBackup> BackupImpl::getBaseBackupUnlocked() const
params.is_internal_backup = is_internal_backup;
/// use_same_s3_credentials_for_base_backup should be inherited for base backups
params.use_same_s3_credentials_for_base_backup = use_same_s3_credentials_for_base_backup;
/// use_same_password_for_base_backup should be inherited for base backups
params.use_same_password_for_base_backup = use_same_password_for_base_backup;
if (params.use_same_password_for_base_backup)
params.password = archive_params.password;
base_backup = BackupFactory::instance().createBackup(params);

7
src/Backups/BackupImpl.h
View File

@ -41,7 +41,8 @@ public:
std::shared_ptr<IBackupReader> reader_,
const ContextPtr & context_,
bool is_internal_backup_,
bool use_same_s3_credentials_for_base_backup_);
bool use_same_s3_credentials_for_base_backup_,
bool use_same_password_for_base_backup_);
BackupImpl(
const BackupInfo & backup_info_,
@ -53,7 +54,8 @@ public:
const std::shared_ptr<IBackupCoordination> & coordination_,
const std::optional<UUID> & backup_uuid_,
bool deduplicate_files_,
bool use_same_s3_credentials_for_base_backup_);
bool use_same_s3_credentials_for_base_backup_,
bool use_same_password_for_base_backup_);
~BackupImpl() override;
@ -153,6 +155,7 @@ private:
bool writing_finalized = false;
bool deduplicate_files = true;
bool use_same_s3_credentials_for_base_backup = false;
bool use_same_password_for_base_backup = false;
const LoggerPtr log;
};

1
src/Backups/BackupSettings.cpp
View File

@ -29,6 +29,7 @@ namespace ErrorCodes
M(Bool, allow_s3_native_copy) \
M(Bool, allow_azure_native_copy) \
M(Bool, use_same_s3_credentials_for_base_backup) \
M(Bool, use_same_password_for_base_backup) \
M(Bool, azure_attempt_to_create_container) \
M(Bool, read_from_filesystem_cache) \
M(UInt64, shard_num) \

3
src/Backups/BackupSettings.h
View File

@ -50,6 +50,9 @@ struct BackupSettings
/// Whether base backup to S3 should inherit credentials from the BACKUP query.
bool use_same_s3_credentials_for_base_backup = false;
/// Whether base backup archive should be unlocked using the same password as the incremental archive
bool use_same_password_for_base_backup = false;
/// Whether a new Azure container should be created if it does not exist (requires permissions at storage account level)
bool azure_attempt_to_create_container = true;

2
src/Backups/BackupsWorker.cpp
View File

@ -602,6 +602,7 @@ void BackupsWorker::doBackup(
backup_create_params.allow_s3_native_copy = backup_settings.allow_s3_native_copy;
backup_create_params.allow_azure_native_copy = backup_settings.allow_azure_native_copy;
backup_create_params.use_same_s3_credentials_for_base_backup = backup_settings.use_same_s3_credentials_for_base_backup;
backup_create_params.use_same_password_for_base_backup = backup_settings.use_same_password_for_base_backup;
backup_create_params.azure_attempt_to_create_container = backup_settings.azure_attempt_to_create_container;
backup_create_params.read_settings = getReadSettingsForBackup(context, backup_settings);
backup_create_params.write_settings = getWriteSettingsForBackup(context);
@ -924,6 +925,7 @@ void BackupsWorker::doRestore(
backup_open_params.password = restore_settings.password;
backup_open_params.allow_s3_native_copy = restore_settings.allow_s3_native_copy;
backup_open_params.use_same_s3_credentials_for_base_backup = restore_settings.use_same_s3_credentials_for_base_backup;
backup_open_params.use_same_password_for_base_backup = restore_settings.use_same_password_for_base_backup;
backup_open_params.read_settings = getReadSettingsForRestore(context);
backup_open_params.write_settings = getWriteSettingsForRestore(context);
backup_open_params.is_internal_backup = restore_settings.internal;

1
src/Backups/RestoreSettings.cpp
View File

@ -164,6 +164,7 @@ namespace
M(RestoreUDFCreationMode, create_function) \
M(Bool, allow_s3_native_copy) \
M(Bool, use_same_s3_credentials_for_base_backup) \
M(Bool, use_same_password_for_base_backup) \
M(Bool, restore_broken_parts_as_detached) \
M(Bool, internal) \
M(String, host_id) \

3
src/Backups/RestoreSettings.h
View File

@ -113,6 +113,9 @@ struct RestoreSettings
/// Whether base backup from S3 should inherit credentials from the RESTORE query.
bool use_same_s3_credentials_for_base_backup = false;
/// Whether base backup archive should be unlocked using the same password as the incremental archive
bool use_same_password_for_base_backup = false;
/// If it's true RESTORE won't stop on broken parts while restoring, instead they will be restored as detached parts
/// to the `detached` folder with names starting with `broken-from-backup'.
bool restore_broken_parts_as_detached = false;

6
src/Backups/registerBackupEngineAzureBlobStorage.cpp
View File

@ -141,7 +141,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
reader,
params.context,
params.is_internal_backup,
/* use_same_s3_credentials_for_base_backup*/ false);
/* use_same_s3_credentials_for_base_backup*/ false,
params.use_same_password_for_base_backup);
}
else
{
@ -164,7 +165,8 @@ void registerBackupEngineAzureBlobStorage(BackupFactory & factory)
params.backup_coordination,
params.backup_uuid,
params.deduplicate_files,
/* use_same_s3_credentials_for_base_backup */ false);
/* use_same_s3_credentials_for_base_backup */ false,
params.use_same_password_for_base_backup);
}
#else
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "AzureBlobStorage support is disabled");

6
src/Backups/registerBackupEngineS3.cpp
View File

@ -120,7 +120,8 @@ void registerBackupEngineS3(BackupFactory & factory)
reader,
params.context,
params.is_internal_backup,
params.use_same_s3_credentials_for_base_backup);
params.use_same_s3_credentials_for_base_backup,
params.use_same_password_for_base_backup);
}
else
{
@ -144,7 +145,8 @@ void registerBackupEngineS3(BackupFactory & factory)
params.backup_coordination,
params.backup_uuid,
params.deduplicate_files,
params.use_same_s3_credentials_for_base_backup);
params.use_same_s3_credentials_for_base_backup,
params.use_same_password_for_base_backup);
}
#else
throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "S3 support is disabled");

6
src/Backups/registerBackupEnginesFileAndDisk.cpp
View File

@ -178,7 +178,8 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
reader,
params.context,
params.is_internal_backup,
params.use_same_s3_credentials_for_base_backup);
params.use_same_s3_credentials_for_base_backup,
params.use_same_password_for_base_backup);
}
else
{
@ -197,7 +198,8 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
params.backup_coordination,
params.backup_uuid,
params.deduplicate_files,
params.use_same_s3_credentials_for_base_backup);
params.use_same_s3_credentials_for_base_backup,
params.use_same_password_for_base_backup);
}
};

21
tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.reference Normal file
View File

@ -0,0 +1,21 @@
use_same_password_for_base_backup
base
BACKUP_CREATED
add_more_data_1
inc_1
BACKUP_CREATED
add_more_data_2
inc_2
BACKUP_CREATED
inc_2_bad
Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_default_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
restore_inc_1
RESTORED
restore_inc_2
RESTORED
restore_inc_2_bad
Couldn't unpack zip archive '02843_backup_use_same_password_for_base_backup_default_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)
count_inc_1
20
count_inc_2
30

50
tests/queries/0_stateless/02843_backup_use_same_password_for_base_backup.sh Executable file
View File

@ -0,0 +1,50 @@
#!/usr/bin/env bash
# Tags: no-fasttest
CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
# shellcheck source=../shell_config.sh
. "$CUR_DIR"/../shell_config.sh
$CLICKHOUSE_CLIENT -nm -q "
DROP TABLE IF EXISTS data;
DROP TABLE IF EXISTS data_1;
DROP TABLE IF EXISTS data_2;
CREATE TABLE data (key Int) ENGINE=MergeTree() ORDER BY tuple();
INSERT INTO data SELECT * from numbers(10);
"
echo 'use_same_password_for_base_backup'
echo "base"
$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_base.zip') SETTINGS password='password';" | cut -f2
echo 'add_more_data_1'
$CLICKHOUSE_CLIENT -q "INSERT INTO data SELECT * FROM numbers(10,10);"
echo "inc_1"
$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_base.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
echo 'add_more_data_2'
$CLICKHOUSE_CLIENT -q "INSERT INTO data SELECT * FROM numbers(20,10);"
echo "inc_2"
$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password',use_same_password_for_base_backup=1" | cut -f2
echo "inc_2_bad"
$CLICKHOUSE_CLIENT -q "BACKUP TABLE data TO Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2_bad.zip') SETTINGS base_backup=Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip'),password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
echo "restore_inc_1"
$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_1 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
echo "restore_inc_2"
$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password',use_same_password_for_base_backup=1" | cut -f2
echo "restore_inc_2_bad"
$CLICKHOUSE_CLIENT -q "RESTORE TABLE data AS data_2 FROM Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_2.zip') SETTINGS password='password'" |& grep -m1 -o "Couldn't unpack zip archive '${CLICKHOUSE_TEST_UNIQUE_NAME}_inc_1.zip': Password is required. (CANNOT_UNPACK_ARCHIVE)"
echo "count_inc_1"
$CLICKHOUSE_CLIENT -q "SELECT COUNT(*) FROM data_1" | cut -f2
echo "count_inc_2"
$CLICKHOUSE_CLIENT -q "SELECT COUNT(*) FROM data_2" | cut -f2
exit 0