thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-21 15:12:02 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #26465 from vitlibar/explicit-encryption-mode

Browse Source 
Set encryption algorithm explicitly.
This commit is contained in:
Vitaly Baranov 2021-07-19 09:38:02 +03:00 committed by GitHub
commit 1fc822b19d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 305 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

104
src/Disks/DiskEncrypted.cpp
View File

@ -22,7 +22,9 @@ namespace
using DiskEncryptedPtr = std::shared_ptr<DiskEncrypted>;
using namespace FileEncryption;
String unhexKey(const String & hex, const String & disk_name)
constexpr Algorithm DEFAULT_ENCRYPTION_ALGORITHM = Algorithm::AES_128_CTR;
String unhexKey(const String & hex)
{
try
{
@ -30,12 +32,13 @@ namespace
}
catch (const std::exception &)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read key_hex for disk {}, check for valid characters [0-9a-fA-F] and length", disk_name);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot read key_hex, check for valid characters [0-9a-fA-F] and length");
}
}
struct DiskEncryptedSettings
{
Algorithm encryption_algorithm;
String key;
DiskPtr wrapped_disk;
String path_on_wrapped_disk;
@ -43,43 +46,46 @@ namespace
DiskEncryptedSettings(
const String & disk_name, const Poco::Util::AbstractConfiguration & config, const String & config_prefix, const DisksMap & map)
{
String wrapped_disk_name = config.getString(config_prefix + ".disk", "");
if (wrapped_disk_name.empty())
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Name of the wrapped disk must not be empty. An encrypted disk is a wrapper over another disk. "
"Disk {}", disk_name);
key = config.getString(config_prefix + ".key", "");
String key_hex = config.getString(config_prefix + ".key_hex", "");
if (!key.empty() && !key_hex.empty())
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Both 'key' and 'key_hex' are specified. There should be only one. Disk {}", disk_name);
if (!key_hex.empty())
try
{
assert(key.empty());
key = unhexKey(key_hex, disk_name);
encryption_algorithm = DEFAULT_ENCRYPTION_ALGORITHM;
if (config.has(config_prefix + ".algorithm"))
parseFromString(encryption_algorithm, config.getString(config_prefix + ".algorithm"));
key = config.getString(config_prefix + ".key", "");
String key_hex = config.getString(config_prefix + ".key_hex", "");
if (!key.empty() && !key_hex.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Both 'key' and 'key_hex' are specified. There should be only one");
if (!key_hex.empty())
{
assert(key.empty());
key = unhexKey(key_hex);
}
FileEncryption::checkKeySize(encryption_algorithm, key.size());
String wrapped_disk_name = config.getString(config_prefix + ".disk", "");
if (wrapped_disk_name.empty())
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Name of the wrapped disk must not be empty. An encrypted disk is a wrapper over another disk");
auto wrapped_disk_it = map.find(wrapped_disk_name);
if (wrapped_disk_it == map.end())
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"The wrapped disk must have been announced earlier. No disk with name {}",
wrapped_disk_name);
wrapped_disk = wrapped_disk_it->second;
path_on_wrapped_disk = config.getString(config_prefix + ".path", "");
}
catch (Exception & e)
{
e.addMessage("Disk " + disk_name);
throw;
}
if (key.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Key of the encrypted disk must not be empty. Disk {}", disk_name);
if (!FileEncryption::isKeyLengthSupported(key.length()))
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Key length is not supported, supported only keys of length 16, 24, or 32 bytes. Disk {}", disk_name);
auto wrapped_disk_it = map.find(wrapped_disk_name);
if (wrapped_disk_it == map.end())
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"The wrapped disk must have been announced earlier. No disk with name {}. Disk {}",
wrapped_disk_name, disk_name);
wrapped_disk = wrapped_disk_it->second;
path_on_wrapped_disk = config.getString(config_prefix + ".path", "");
}
};
@ -123,9 +129,13 @@ ReservationPtr DiskEncrypted::reserve(UInt64 bytes)
return std::make_unique<DiskEncryptedReservation>(std::static_pointer_cast<DiskEncrypted>(shared_from_this()), std::move(reservation));
}
DiskEncrypted::DiskEncrypted(const String & name_, DiskPtr disk_, const String & key_, const String & path_)
: DiskDecorator(disk_)
, name(name_), key(key_), disk_path(path_)
DiskEncrypted::DiskEncrypted(
const String & name_,
DiskPtr wrapped_disk_,
const String & path_on_wrapped_disk_,
FileEncryption::Algorithm encryption_algorithm_,
const String & key_)
: DiskDecorator(wrapped_disk_), name(name_), disk_path(path_on_wrapped_disk_), encryption_algorithm(encryption_algorithm_), key(key_)
{
initialize();
}
@ -152,10 +162,10 @@ void DiskEncrypted::copy(const String & from_path, const std::shared_ptr<IDisk>
/// Disk type is the same, check if the key is the same too.
if (auto * to_encrypted_disk = typeid_cast<DiskEncrypted *>(to_disk.get()))
{
if (key == to_encrypted_disk->key)
if ((encryption_algorithm == to_encrypted_disk->encryption_algorithm) && (key == to_encrypted_disk->key))
{
/// Key is the same so we can simply copy the encrypted file.
delegate->copy(wrappedPath(from_path), to_encrypted_disk->delegate, wrappedPath(to_path));
delegate->copy(wrappedPath(from_path), to_encrypted_disk->delegate, to_encrypted_disk->wrappedPath(to_path));
return;
}
}
@ -178,7 +188,7 @@ std::unique_ptr<ReadBufferFromFileBase> DiskEncrypted::readFile(
InitVector iv;
iv.read(*buffer);
return std::make_unique<ReadBufferFromEncryptedFile>(buf_size, std::move(buffer), key, iv);
return std::make_unique<ReadBufferFromEncryptedFile>(buf_size, std::move(buffer), encryption_algorithm, key, iv);
}
std::unique_ptr<WriteBufferFromFileBase> DiskEncrypted::writeFile(const String & path, size_t buf_size, WriteMode mode)
@ -197,7 +207,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskEncrypted::writeFile(const String &
iv = InitVector::random();
auto buffer = delegate->writeFile(wrapped_path, buf_size, mode);
return std::make_unique<WriteBufferFromEncryptedFile>(buf_size, std::move(buffer), key, iv, old_file_size);
return std::make_unique<WriteBufferFromEncryptedFile>(buf_size, std::move(buffer), encryption_algorithm, key, iv, old_file_size);
}
@ -227,9 +237,10 @@ void DiskEncrypted::applyNewSettings(
const DisksMap & map)
{
DiskEncryptedSettings settings{name, config, config_prefix, map};
key = settings.key;
delegate = settings.wrapped_disk;
disk_path = settings.path_on_wrapped_disk;
encryption_algorithm = settings.encryption_algorithm;
key = settings.key;
initialize();
}
@ -242,7 +253,8 @@ void registerDiskEncrypted(DiskFactory & factory)
const DisksMap & map) -> DiskPtr
{
DiskEncryptedSettings settings{name, config, config_prefix, map};
return std::make_shared<DiskEncrypted>(name, settings.wrapped_disk, settings.key, settings.path_on_wrapped_disk);
return std::make_shared<DiskEncrypted>(
name, settings.wrapped_disk, settings.path_on_wrapped_disk, settings.encryption_algorithm, settings.key);
};
factory.registerDiskType("encrypted", creator);
}

11
src/Disks/DiskEncrypted.h
View File

@ -13,6 +13,7 @@ namespace DB
{
class ReadBufferFromFileBase;
class WriteBufferFromFileBase;
namespace FileEncryption { enum class Algorithm; }
/// Encrypted disk ciphers all written files on the fly and writes the encrypted files to an underlying (normal) disk.
/// And when we read files from an encrypted disk it deciphers them automatically,
@ -20,7 +21,12 @@ class WriteBufferFromFileBase;
class DiskEncrypted : public DiskDecorator
{
public:
DiskEncrypted(const String & name_, DiskPtr disk_, const String & key_, const String & path_);
DiskEncrypted(
const String & name_,
DiskPtr wrapped_disk_,
const String & path_on_wrapped_disk_,
FileEncryption::Algorithm encryption_algorithm_,
const String & key_);
const String & getName() const override { return name; }
const String & getPath() const override { return disk_absolute_path; }
@ -219,9 +225,10 @@ private:
}
String name;
String key;
String disk_path;
String disk_absolute_path;
FileEncryption::Algorithm encryption_algorithm;
String key;
};
}

93
src/IO/FileEncryptionCommon.cpp
View File

@ -6,6 +6,7 @@
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <boost/algorithm/string/predicate.hpp>
#include <cassert>
#include <random>
@ -15,6 +16,7 @@ namespace DB
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int DATA_ENCRYPTION_ERROR;
}
@ -23,6 +25,41 @@ namespace FileEncryption
namespace
{
const EVP_CIPHER * getCipher(Algorithm algorithm)
{
switch (algorithm)
{
case Algorithm::AES_128_CTR: return EVP_aes_128_ctr();
case Algorithm::AES_192_CTR: return EVP_aes_192_ctr();
case Algorithm::AES_256_CTR: return EVP_aes_256_ctr();
}
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Encryption algorithm {} is not supported, specify one of the following: aes_128_ctr, aes_192_ctr, aes_256_ctr",
std::to_string(static_cast<int>(algorithm)));
}
void checkKeySize(const EVP_CIPHER * evp_cipher, size_t key_size)
{
if (!key_size)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Encryption key must not be empty");
size_t expected_key_size = static_cast<size_t>(EVP_CIPHER_key_length(evp_cipher));
if (key_size != expected_key_size)
throw Exception(
ErrorCodes::BAD_ARGUMENTS, "Got an encryption key with unexpected size {}, the size should be {}", key_size, expected_key_size);
}
void checkInitVectorSize(const EVP_CIPHER * evp_cipher)
{
size_t expected_iv_length = static_cast<size_t>(EVP_CIPHER_iv_length(evp_cipher));
if (InitVector::kSize != expected_iv_length)
throw Exception(
ErrorCodes::DATA_ENCRYPTION_ERROR,
"Got an initialization vector with unexpected size {}, the size should be {}",
InitVector::kSize,
expected_iv_length);
}
constexpr const size_t kBlockSize = 16;
size_t blockOffset(size_t pos) { return pos % kBlockSize; }
@ -145,6 +182,39 @@ namespace
}
}
String toString(Algorithm algorithm)
{
switch (algorithm)
{
case Algorithm::AES_128_CTR: return "aes_128_ctr";
case Algorithm::AES_192_CTR: return "aes_192_ctr";
case Algorithm::AES_256_CTR: return "aes_256_ctr";
}
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Encryption algorithm {} is not supported, specify one of the following: aes_128_ctr, aes_192_ctr, aes_256_ctr",
std::to_string(static_cast<int>(algorithm)));
}
void parseFromString(Algorithm & algorithm, const String & str)
{
if (boost::iequals(str, "aes_128_ctr"))
algorithm = Algorithm::AES_128_CTR;
else if (boost::iequals(str, "aes_192_ctr"))
algorithm = Algorithm::AES_192_CTR;
else if (boost::iequals(str, "aes_256_ctr"))
algorithm = Algorithm::AES_256_CTR;
else
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Encryption algorithm '{}' is not supported, specify one of the following: aes_128_ctr, aes_192_ctr, aes_256_ctr",
str);
}
void checkKeySize(Algorithm algorithm, size_t key_size) { checkKeySize(getCipher(algorithm), key_size); }
String InitVector::toString() const
{
static_assert(sizeof(counter) == InitVector::kSize);
@ -156,7 +226,7 @@ String InitVector::toString() const
InitVector InitVector::fromString(const String & str)
{
if (str.length() != InitVector::kSize)
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Expected iv with size {}, got iv with size {}", InitVector::kSize, str.length());
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected iv with size {}, got iv with size {}", InitVector::kSize, str.length());
ReadBufferFromMemory in{str.data(), str.length()};
UInt128 counter;
readBinaryBigEndian(counter, in);
@ -185,26 +255,13 @@ InitVector InitVector::random()
}
Encryptor::Encryptor(const String & key_, const InitVector & iv_)
Encryptor::Encryptor(Algorithm algorithm_, const String & key_, const InitVector & iv_)
: key(key_)
, init_vector(iv_)
, evp_cipher(getCipher(algorithm_))
{
if (key_.length() == 16)
evp_cipher = EVP_aes_128_ctr();
else if (key_.length() == 24)
evp_cipher = EVP_aes_192_ctr();
else if (key_.length() == 32)
evp_cipher = EVP_aes_256_ctr();
else
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Key length {} is not supported, supported only keys of length 128, 192, or 256 bits", key_.length());
size_t cipher_key_length = static_cast<size_t>(EVP_CIPHER_key_length(evp_cipher));
if (cipher_key_length != key_.length())
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Got unexpected key length from cipher: {} != {}", cipher_key_length, key_.length());
size_t cipher_iv_length = static_cast<size_t>(EVP_CIPHER_iv_length(evp_cipher));
if (cipher_iv_length != InitVector::kSize)
throw Exception(ErrorCodes::DATA_ENCRYPTION_ERROR, "Got unexpected init vector's length from cipher: {} != {}", cipher_iv_length, InitVector::kSize);
checkKeySize(evp_cipher, key.size());
checkInitVectorSize(evp_cipher);
}
void Encryptor::encrypt(const char * data, size_t size, WriteBuffer & out)

34
src/IO/FileEncryptionCommon.h
View File

@ -16,6 +16,24 @@ class WriteBuffer;
namespace FileEncryption
{
/// Encryption algorithm.
/// We chose to use CTR cipther algorithms because they have the following features which are important for us:
/// - No right padding, so we can append encrypted files without deciphering;
/// - One byte is always ciphered as one byte, so we get random access to encrypted files easily.
enum class Algorithm
{
AES_128_CTR, /// Size of key is 16 bytes.
AES_192_CTR, /// Size of key is 24 bytes.
AES_256_CTR, /// Size of key is 32 bytes.
};
String toString(Algorithm algorithm);
void parseFromString(Algorithm & algorithm, const String & str);
/// Throws an exception if a specified key size doesn't correspond a specified encryption algorithm.
void checkKeySize(Algorithm algorithm, size_t key_size);
/// Initialization vector. Its size is always 16 bytes.
class InitVector
{
@ -51,17 +69,12 @@ private:
UInt128 counter = 0;
};
/// Encrypts or decrypts data.
class Encryptor
{
public:
/// The `key` should have length 128 or 192 or 256.
/// According to the key's length aes_128_ctr or aes_192_ctr or aes_256_ctr will be used for encryption.
/// We chose to use CTR cipther algorithms because they have the following features which are important for us:
/// - No right padding, so we can append encrypted files without deciphering;
/// - One byte is always ciphered as one byte, so we get random access to encrypted files easily.
Encryptor(const String & key_, const InitVector & iv_);
/// The `key` should have size 16 or 24 or 32 bytes depending on which `algorithm` is specified.
Encryptor(Algorithm algorithm_, const String & key_, const InitVector & iv_);
/// Sets the current position in the data stream from the very beginning of data.
/// It affects how the data will be encrypted or decrypted because
@ -82,17 +95,12 @@ public:
private:
const String key;
const InitVector init_vector;
const EVP_CIPHER * evp_cipher;
const EVP_CIPHER * const evp_cipher;
/// The current position in the data stream from the very beginning of data.
size_t offset = 0;
};
/// Checks whether a passed key length is supported, i.e.
/// whether its length is 128 or 192 or 256 bits (16 or 24 or 32 bytes).
bool isKeyLengthSupported(size_t key_length);
}
}

3
src/IO/ReadBufferFromEncryptedFile.cpp
View File

@ -14,12 +14,13 @@ using InitVector = FileEncryption::InitVector;
ReadBufferFromEncryptedFile::ReadBufferFromEncryptedFile(
size_t buffer_size_,
std::unique_ptr<ReadBufferFromFileBase> in_,
FileEncryption::Algorithm encryption_algorithm_,
const String & key_,
const InitVector & init_vector_)
: ReadBufferFromFileBase(buffer_size_, nullptr, 0)
, in(std::move(in_))
, encrypted_buffer(buffer_size_)
, encryptor(key_, init_vector_)
, encryptor(encryption_algorithm_, key_, init_vector_)
{
/// We should start reading from `in` at the offset == InitVector::kSize.
need_seek = true;

1
src/IO/ReadBufferFromEncryptedFile.h
View File

@ -19,6 +19,7 @@ public:
ReadBufferFromEncryptedFile(
size_t buffer_size_,
std::unique_ptr<ReadBufferFromFileBase> in_,
FileEncryption::Algorithm encryption_algorithm_,
const String & key_,
const FileEncryption::InitVector & init_vector_);

3
src/IO/WriteBufferFromEncryptedFile.cpp
View File

@ -11,6 +11,7 @@ using InitVector = FileEncryption::InitVector;
WriteBufferFromEncryptedFile::WriteBufferFromEncryptedFile(
size_t buffer_size_,
std::unique_ptr<WriteBufferFromFileBase> out_,
FileEncryption::Algorithm encryption_algorithm_,
const String & key_,
const InitVector & init_vector_,
size_t old_file_size)
@ -18,7 +19,7 @@ WriteBufferFromEncryptedFile::WriteBufferFromEncryptedFile(
, out(std::move(out_))
, iv(init_vector_)
, flush_iv(!old_file_size)
, encryptor(key_, init_vector_)
, encryptor(encryption_algorithm_, key_, init_vector_)
{
encryptor.setOffset(old_file_size);
}

1
src/IO/WriteBufferFromEncryptedFile.h
View File

@ -20,6 +20,7 @@ public:
WriteBufferFromEncryptedFile(
size_t buffer_size_,
std::unique_ptr<WriteBufferFromFileBase> out_,
FileEncryption::Algorithm encryption_algorithm_,
const String & key_,
const FileEncryption::InitVector & init_vector_,
size_t old_file_size = 0);

160
src/IO/tests/gtest_file_encryption.cpp
View File

@ -11,26 +11,18 @@
using namespace DB;
using namespace DB::FileEncryption;
struct InitVectorTestParam
{
const std::string_view comment;
const String init;
const String after_inc;
UInt64 adder;
const UInt64 adder;
const String after_add;
};
class FileEncryptionInitVectorTest : public ::testing::TestWithParam<InitVectorTestParam> {};
class InitVectorTest : public ::testing::TestWithParam<InitVectorTestParam> {};
static std::ostream & operator << (std::ostream & ostr, const InitVectorTestParam & param)
{
return ostr << param.comment;
}
TEST_P(InitVectorTest, InitVector)
TEST_P(FileEncryptionInitVectorTest, InitVector)
{
const auto & param = GetParam();
@ -44,33 +36,32 @@ TEST_P(InitVectorTest, InitVector)
ASSERT_EQ(param.after_add, iv.toString());
}
INSTANTIATE_TEST_SUITE_P(InitVectorInputs,
InitVectorTest,
::testing::ValuesIn(std::initializer_list<InitVectorTestParam>{
{
"Basic init vector test. Get zero-string, add 1, add 0",
INSTANTIATE_TEST_SUITE_P(All,
FileEncryptionInitVectorTest,
::testing::ValuesIn(std::initializer_list<InitVectorTestParam>
{
{ // #0. Basic init vector test. Get zero-string, add 1, add 0.
String(16, 0),
String("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01", 16),
0,
String("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01", 16),
},
{
"Init vector test. Get zero-string, add 1, add 85, add 1024",
// #1. Init vector test. Get zero-string, add 1, add 85, add 1024.
String(16, 0),
String("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01", 16),
85,
String("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x56", 16),
},
{
"Init vector test #2. Get zero-string, add 1, add 1024",
// #2. Init vector test #2. Get zero-string, add 1, add 1024.
String(16, 0),
String("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x01", 16),
1024,
String("\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x04\x01", 16)
},
{
"Long init vector test",
// #3. Long init vector test.
String("\xa8\x65\x9c\x73\xf8\x5d\x83\xb4\x9c\xa6\x8c\x19\xf4\x77\x80\xe1", 16),
String("\xa8\x65\x9c\x73\xf8\x5d\x83\xb4\x9c\xa6\x8c\x19\xf4\x77\x80\xe2", 16),
9349249176525638641ULL,
@ -80,15 +71,28 @@ INSTANTIATE_TEST_SUITE_P(InitVectorInputs,
);
TEST(FileEncryption, Encryption)
struct CipherTestParam
{
String key = "1234567812345678";
InitVector iv;
Encryptor encryptor{key, iv};
const Algorithm algorithm;
const String key;
const InitVector iv;
const size_t offset;
const String plaintext;
const String ciphertext;
};
std::string_view input = "abcd1234efgh5678ijkl";
std::string_view expected = "\xfb\x8a\x9e\x66\x82\x72\x1b\xbe\x6b\x1d\xd8\x98\xc5\x8c\x63\xee\xcd\x36\x4a\x50";
class FileEncryptionCipherTest : public ::testing::TestWithParam<CipherTestParam> {};
TEST_P(FileEncryptionCipherTest, Encryption)
{
const auto & param = GetParam();
Encryptor encryptor{param.algorithm, param.key, param.iv};
std::string_view input = param.plaintext;
std::string_view expected = param.ciphertext;
size_t base_offset = param.offset;
encryptor.setOffset(base_offset);
for (size_t i = 0; i < expected.size(); ++i)
{
WriteBufferFromOwnString buf;
@ -99,7 +103,7 @@ TEST(FileEncryption, Encryption)
for (size_t i = 0; i < expected.size(); ++i)
{
WriteBufferFromOwnString buf;
encryptor.setOffset(i);
encryptor.setOffset(base_offset + i);
encryptor.encrypt(&input[i], 1, buf);
ASSERT_EQ(expected.substr(i, 1), buf.str());
}
@ -107,32 +111,22 @@ TEST(FileEncryption, Encryption)
for (size_t i = 0; i <= expected.size(); ++i)
{
WriteBufferFromOwnString buf;
encryptor.setOffset(0);
encryptor.setOffset(base_offset);
encryptor.encrypt(input.data(), i, buf);
ASSERT_EQ(expected.substr(0, i), buf.str());
}
size_t offset = 25;
std::string_view offset_expected = "\x6c\x67\xe4\xf5\x8f\x86\xb0\x19\xe5\xcd\x53\x59\xe0\xc6\x01\x5e\xc1\xfd\x60\x9d";
for (size_t i = 0; i <= expected.size(); ++i)
{
WriteBufferFromOwnString buf;
encryptor.setOffset(offset);
encryptor.encrypt(input.data(), i, buf);
ASSERT_EQ(offset_expected.substr(0, i), buf.str());
}
}
TEST(FileEncryption, Decryption)
TEST_P(FileEncryptionCipherTest, Decryption)
{
String key("1234567812345678");
InitVector iv;
Encryptor encryptor{key, iv};
const auto & param = GetParam();
std::string_view input = "\xfb\x8a\x9e\x66\x82\x72\x1b\xbe\x6b\x1d\xd8\x98\xc5\x8c\x63\xee\xcd\x36\x4a\x50";
std::string_view expected = "abcd1234efgh5678ijkl";
Encryptor encryptor{param.algorithm, param.key, param.iv};
std::string_view input = param.ciphertext;
std::string_view expected = param.plaintext;
size_t base_offset = param.offset;
encryptor.setOffset(base_offset);
for (size_t i = 0; i < expected.size(); ++i)
{
char c;
@ -143,7 +137,7 @@ TEST(FileEncryption, Decryption)
for (size_t i = 0; i < expected.size(); ++i)
{
char c;
encryptor.setOffset(i);
encryptor.setOffset(base_offset + i);
encryptor.decrypt(&input[i], 1, &c);
ASSERT_EQ(expected[i], c);
}
@ -151,19 +145,71 @@ TEST(FileEncryption, Decryption)
String buf(expected.size(), 0);
for (size_t i = 0; i <= expected.size(); ++i)
{
encryptor.setOffset(0);
encryptor.setOffset(base_offset);
encryptor.decrypt(input.data(), i, buf.data());
ASSERT_EQ(expected.substr(0, i), buf.substr(0, i));
}
size_t offset = 25;
String offset_input = "\x6c\x67\xe4\xf5\x8f\x86\xb0\x19\xe5\xcd\x53\x59\xe0\xc6\x01\x5e\xc1\xfd\x60\x9d";
for (size_t i = 0; i <= expected.size(); ++i)
{
encryptor.setOffset(offset);
encryptor.decrypt(offset_input.data(), i, buf.data());
ASSERT_EQ(expected.substr(0, i), buf.substr(0, i));
}
}
INSTANTIATE_TEST_SUITE_P(All,
FileEncryptionCipherTest,
::testing::ValuesIn(std::initializer_list<CipherTestParam>
{
{
// #0
Algorithm::AES_128_CTR,
"1234567812345678",
InitVector{},
0,
"abcd1234efgh5678ijkl",
"\xfb\x8a\x9e\x66\x82\x72\x1b\xbe\x6b\x1d\xd8\x98\xc5\x8c\x63\xee\xcd\x36\x4a\x50"
},
{
// #1
Algorithm::AES_128_CTR,
"1234567812345678",
InitVector{},
25,
"abcd1234efgh5678ijkl",
"\x6c\x67\xe4\xf5\x8f\x86\xb0\x19\xe5\xcd\x53\x59\xe0\xc6\x01\x5e\xc1\xfd\x60\x9d"
},
{
// #2
Algorithm::AES_128_CTR,
String{"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 16},
InitVector{},
0,
"abcd1234efgh5678ijkl",
"\xa7\xc3\x58\x53\xb6\xbd\x68\xb6\x0a\x29\xe6\x0a\x94\xfe\xef\x41\x1a\x2c\x78\xf9"
},
{
// #3
Algorithm::AES_128_CTR,
"1234567812345678",
InitVector::fromString(String{"\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f", 16}),
0,
"abcd1234efgh5678ijkl",
"\xcf\xab\x7c\xad\xa9\xdc\x67\x60\x90\x85\x7b\xb8\x72\xa9\x6f\x9c\x29\xb2\x4f\xf6"
},
{
// #4
Algorithm::AES_192_CTR,
"123456781234567812345678",
InitVector{},
0,
"abcd1234efgh5678ijkl",
"\xcc\x25\x2b\xad\xe8\xa2\xdc\x64\x3e\xf9\x60\xe0\x6e\xde\x70\xb6\x63\xa8\xfa\x02"
},
{
// #5
Algorithm::AES_256_CTR,
"12345678123456781234567812345678",
InitVector{},
0,
"abcd1234efgh5678ijkl",
"\xc7\x41\xa6\x63\x04\x60\x1b\x1a\xcb\x84\x19\xce\x3a\x36\xa3\xbd\x21\x71\x93\xfb"
},
})
);
#endif

24
tests/integration/test_encrypted_disk/configs/storage.xml
View File

@ -25,8 +25,21 @@
<type>encrypted</type>
<disk>disk_local</disk>
<path>encrypted/</path>
<key_hex>109105c600c12066f82f1a4dbb41a08e</key_hex>
<key>1234567812345678</key>
</disk_local_encrypted>
<disk_local_encrypted2>
<type>encrypted</type>
<disk>disk_local</disk>
<path>encrypted2/</path>
<key>1234567812345678</key>
</disk_local_encrypted2>
<disk_local_encrypted_key192b>
<type>encrypted</type>
<disk>disk_local</disk>
<path>encrypted_key192b/</path>
<algorithm>aes_192_ctr</algorithm>
<key_hex>109105c600c12066f82f1a4dbb41a08e4A4348C8387ADB6A</key_hex>
</disk_local_encrypted_key192b>
</disks>
<policies>
<encrypted_policy>
@ -36,6 +49,13 @@
</main>
</volumes>
</encrypted_policy>
<encrypted_policy_key192b>
<volumes>
<main>
<disk>disk_local_encrypted_key192b</disk>
</main>
</volumes>
</encrypted_policy_key192b>
<local_policy>
<volumes>
<main>
@ -43,6 +63,8 @@
</main>
<external>
<disk>disk_local_encrypted</disk>
<disk>disk_local_encrypted2</disk>
<disk>disk_local_encrypted_key192b</disk>
</external>
</volumes>
</local_policy>

23
tests/integration/test_encrypted_disk/test.py
View File

@ -19,7 +19,7 @@ def cluster():
cluster.shutdown()
@pytest.mark.parametrize("policy", ["encrypted_policy", "local_policy", "s3_policy"])
@pytest.mark.parametrize("policy", ["encrypted_policy", "encrypted_policy_key192b", "local_policy", "s3_policy"])
def test_encrypted_disk(cluster, policy):
node = cluster.instances["node"]
node.query(
@ -44,8 +44,8 @@ def test_encrypted_disk(cluster, policy):
node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY")
@pytest.mark.parametrize("policy,disk,encrypted_disk", [("local_policy", "disk_local", "disk_local_encrypted"), ("s3_policy", "disk_s3", "disk_s3_encrypted")])
def test_part_move(cluster, policy, disk, encrypted_disk):
@pytest.mark.parametrize("policy, destination_disks", [("local_policy", ["disk_local_encrypted", "disk_local_encrypted2", "disk_local_encrypted_key192b", "disk_local"]), ("s3_policy", ["disk_s3_encrypted", "disk_s3"])])
def test_part_move(cluster, policy, destination_disks):
node = cluster.instances["node"]
node.query(
"""
@ -62,17 +62,14 @@ def test_part_move(cluster, policy, disk, encrypted_disk):
select_query = "SELECT * FROM encrypted_test ORDER BY id FORMAT Values"
assert node.query(select_query) == "(0,'data'),(1,'data')"
node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, encrypted_disk))
for destination_disk in destination_disks:
node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, destination_disk))
assert node.query(select_query) == "(0,'data'),(1,'data')"
with pytest.raises(QueryRuntimeException) as exc:
node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, destination_disk))
assert("Part '{}' is already on disk '{}'".format(FIRST_PART_NAME, destination_disk) in str(exc.value))
assert node.query(select_query) == "(0,'data'),(1,'data')"
with pytest.raises(QueryRuntimeException) as exc:
node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, encrypted_disk))
assert("Part '{}' is already on disk '{}'".format(FIRST_PART_NAME, encrypted_disk) in str(exc.value))
node.query("ALTER TABLE encrypted_test MOVE PART '{}' TO DISK '{}'".format(FIRST_PART_NAME, disk))
assert node.query(select_query) == "(0,'data'),(1,'data')"
node.query("DROP TABLE IF EXISTS encrypted_test NO DELAY")