mirror of
https://github.com/ClickHouse/ClickHouse.git
synced 2024-09-29 21:20:49 +00:00
remove debug information, fix seek issue and encryption issue
This commit is contained in:
parent
10a5b734ee
commit
af420569f8
@ -62,15 +62,13 @@ std::unique_ptr<ReadBufferFromFileBase> DiskEncrypted::readFile(
|
||||
size_t mmap_threshold,
|
||||
MMappedFileCache * mmap_cache) const
|
||||
{
|
||||
using namespace FileEncryption;
|
||||
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
auto buffer = delegate->readFile(wrapped_path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
|
||||
|
||||
auto iv = GetRandomString(kIVSize);
|
||||
size_t offset = 0;
|
||||
|
||||
if (delegate->getFileSize(wrapped_path))
|
||||
if (exists(path) && getFileSize(path))
|
||||
{
|
||||
iv = ReadIV(kIVSize, *buffer);
|
||||
offset = kIVSize;
|
||||
@ -84,16 +82,14 @@ std::unique_ptr<WriteBufferFromFileBase> DiskEncrypted::writeFile(
|
||||
size_t buf_size,
|
||||
WriteMode mode)
|
||||
{
|
||||
using namespace FileEncryption;
|
||||
|
||||
auto wrapped_path = wrappedPath(path);
|
||||
auto iv = GetRandomString(kIVSize);
|
||||
|
||||
try {
|
||||
if (exists(path) && getFileSize(path))
|
||||
{
|
||||
auto read_buffer = delegate->readFile(wrapped_path, kIVSize);
|
||||
iv = ReadIV(kIVSize, *read_buffer);
|
||||
}
|
||||
catch ( ... ) { }
|
||||
|
||||
auto buffer = delegate->writeFile(wrapped_path, buf_size, mode);
|
||||
return std::make_unique<WriteEncryptedBuffer>(buf_size, std::move(buffer), iv, key,
|
||||
@ -161,6 +157,9 @@ void registerDiskEncrypted(DiskFactory & factory)
|
||||
String key = config.getString(config_prefix + ".key", "");
|
||||
if (key.empty())
|
||||
throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
if (key.size() != CipherKeyLength(DefaultCipher()))
|
||||
throw Exception("Expected key with size " + std::to_string(CipherKeyLength(DefaultCipher())) + ", got key with size " + std::to_string(key.size()),
|
||||
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
|
||||
|
||||
auto wrapped_disk = map.find(wrapped_disk_name);
|
||||
if (wrapped_disk == map.end())
|
||||
|
@ -1,7 +1,5 @@
|
||||
#pragma once
|
||||
|
||||
#include <fstream>
|
||||
|
||||
#include <openssl/evp.h>
|
||||
#include <openssl/engine.h>
|
||||
|
||||
@ -9,8 +7,6 @@
|
||||
#include <IO/WriteBuffer.h>
|
||||
#include <IO/WriteHelpers.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
#include <sys/random.h>
|
||||
|
||||
#include <limits>
|
||||
@ -19,10 +15,11 @@
|
||||
|
||||
#include <Functions/FunctionsAES.h>
|
||||
|
||||
|
||||
namespace FileEncryption
|
||||
{
|
||||
|
||||
constexpr size_t kIVSize = 128 / CHAR_BIT / 2;
|
||||
constexpr size_t kIVSize = sizeof(DB::UInt64);
|
||||
|
||||
class InitVector
|
||||
{
|
||||
@ -59,7 +56,6 @@ private:
|
||||
mutable String modify;
|
||||
};
|
||||
|
||||
|
||||
class EncryptionKey
|
||||
{
|
||||
public:
|
||||
@ -87,6 +83,12 @@ String GetRandomString(size_t size)
|
||||
return iv;
|
||||
}
|
||||
|
||||
void WriteIV(const String & iv, DB::WriteBuffer & out)
|
||||
{
|
||||
for (auto i : iv)
|
||||
DB::writeChar(i, out);
|
||||
}
|
||||
|
||||
size_t CipherKeyLength(const EVP_CIPHER * evp_cipher)
|
||||
{
|
||||
return static_cast<size_t>(EVP_CIPHER_key_length(evp_cipher));
|
||||
@ -97,12 +99,16 @@ size_t CipherIVLength(const EVP_CIPHER * evp_cipher)
|
||||
return static_cast<size_t>(EVP_CIPHER_iv_length(evp_cipher));
|
||||
}
|
||||
|
||||
const EVP_CIPHER * DefaultCipher()
|
||||
{
|
||||
return EVP_aes_128_ctr();
|
||||
}
|
||||
|
||||
class Encryption
|
||||
{
|
||||
public:
|
||||
Encryption(const String & iv_, const EncryptionKey & key_, size_t offset_ = 0)
|
||||
: evp_cipher(EVP_aes_128_ctr())
|
||||
: evp_cipher(DefaultCipher())
|
||||
, init_vector(iv_)
|
||||
, key(key_)
|
||||
, block_size(CipherIVLength(evp_cipher))
|
||||
@ -127,13 +133,12 @@ protected:
|
||||
const EVP_CIPHER * get() const { return evp_cipher; }
|
||||
|
||||
const EVP_CIPHER * evp_cipher;
|
||||
const InitVector init_vector;
|
||||
EncryptionKey key;
|
||||
const String init_vector;
|
||||
const EncryptionKey key;
|
||||
size_t block_size;
|
||||
size_t offset = 0; // global offset
|
||||
size_t offset = 0;
|
||||
};
|
||||
|
||||
|
||||
class Encryptor : public Encryption
|
||||
{
|
||||
public:
|
||||
@ -145,22 +150,24 @@ public:
|
||||
if (!size)
|
||||
return;
|
||||
|
||||
auto iv = init_vector;
|
||||
auto iv = InitVector(init_vector);
|
||||
iv.SetCounter(Blocks(offset));
|
||||
size_t first_block_size = FirstBlockSize(size, offset);
|
||||
|
||||
size_t first_block_size = FirstBlockSize(size, offset);
|
||||
if (offset)
|
||||
{
|
||||
buf.write(EncryptPartialBlock(plaintext, first_block_size, iv, offset).data(), first_block_size);
|
||||
offset = BlockOffset(offset + first_block_size);
|
||||
size -= first_block_size;
|
||||
offset += first_block_size;
|
||||
iv.Inc();
|
||||
}
|
||||
|
||||
if (size)
|
||||
{
|
||||
size_t blocks = Blocks(size);
|
||||
buf.write(EncryptNBytes(plaintext + first_block_size, size, iv).data(), size);
|
||||
offset += size;
|
||||
iv.Inc(blocks);
|
||||
offset = size - blocks * block_size;
|
||||
}
|
||||
}
|
||||
|
||||
@ -169,7 +176,7 @@ private:
|
||||
{
|
||||
using namespace OpenSSLDetails;
|
||||
if (size > block_size)
|
||||
onError("Expecter partial block, got block with size > block_size: size = " + std::to_string(size));
|
||||
onError("Expecter partial block, got block with size > block_size: size = " + std::to_string(size) + " and offset = " + std::to_string(off));
|
||||
|
||||
String plaintext(block_size, '\0');
|
||||
for (size_t i = 0; i < size; ++i)
|
||||
@ -204,9 +211,6 @@ private:
|
||||
|
||||
ciphertext_ref += output_len;
|
||||
|
||||
|
||||
//* always do nothing for default cipher
|
||||
|
||||
int final_output_len = 0;
|
||||
if (EVP_EncryptFinal_ex(evp_ctx,
|
||||
reinterpret_cast<unsigned char*>(ciphertext_ref), &final_output_len) != 1)
|
||||
@ -230,7 +234,7 @@ public:
|
||||
if (!size)
|
||||
return;
|
||||
|
||||
auto iv = init_vector;
|
||||
auto iv = InitVector(init_vector);
|
||||
iv.SetCounter(Blocks(off));
|
||||
off = BlockOffset(off);
|
||||
|
||||
@ -239,12 +243,15 @@ public:
|
||||
{
|
||||
DecryptPartialBlock(buf, ciphertext, first_block_size, iv, off);
|
||||
size -= first_block_size;
|
||||
if (first_block_size + off == block_size)
|
||||
iv.Inc();
|
||||
}
|
||||
|
||||
if (size)
|
||||
{
|
||||
size_t blocks = Blocks(size);
|
||||
DecryptNBytes(buf, ciphertext + first_block_size, size, iv);
|
||||
iv.Inc(blocks);
|
||||
}
|
||||
}
|
||||
|
||||
@ -275,7 +282,7 @@ private:
|
||||
auto * evp_ctx = evp_ctx_ptr.get();
|
||||
|
||||
if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1)
|
||||
onError("Failed to initialize decryption context with cipher");
|
||||
onError("Failed to initialize encryption context with cipher");
|
||||
|
||||
if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr,
|
||||
reinterpret_cast<const unsigned char*>(key.GetRef()),
|
||||
@ -290,9 +297,6 @@ private:
|
||||
|
||||
to += output_len;
|
||||
|
||||
|
||||
//* always do nothing for default cipher
|
||||
|
||||
int final_output_len = 0;
|
||||
if (EVP_DecryptFinal_ex(evp_ctx,
|
||||
reinterpret_cast<unsigned char*>(to), &final_output_len) != 1)
|
||||
|
@ -4,7 +4,6 @@
|
||||
#include <IO/WriteBufferFromFileBase.h>
|
||||
#include <Functions/FileEncryption.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -24,7 +23,6 @@ public:
|
||||
, in(std::move(in_))
|
||||
, buf_size(buf_size_)
|
||||
, decryptor(Decryptor(init_vector_, key_))
|
||||
, start_pos(iv_offset_)
|
||||
, iv_offset(iv_offset_)
|
||||
{ }
|
||||
|
||||
@ -35,36 +33,35 @@ public:
|
||||
if (off < 0 && -off > getPosition())
|
||||
throw Exception("SEEK_CUR shift out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
if (!working_buffer.empty() && size_t(offset() + off) < working_buffer.size())
|
||||
if (!working_buffer.empty() && static_cast<size_t>(offset() + off) < working_buffer.size())
|
||||
{
|
||||
pos += off;
|
||||
return getPosition();
|
||||
}
|
||||
else
|
||||
start_pos = off + getPosition() + iv_offset;
|
||||
start_pos = off + getPosition();
|
||||
}
|
||||
else if (whence == SEEK_SET)
|
||||
{
|
||||
if (off < 0)
|
||||
throw Exception("SEEK_SET underflow", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
throw Exception("SEEK_SET underflow: off = " + std::to_string(off), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
if (!working_buffer.empty() && size_t(off) >= start_pos - working_buffer.size()
|
||||
&& size_t(off) < start_pos)
|
||||
if (!working_buffer.empty() && static_cast<size_t>(off) >= start_pos && static_cast<size_t>(off) < (start_pos + working_buffer.size()))
|
||||
{
|
||||
pos = working_buffer.end() - (start_pos - off);
|
||||
pos = working_buffer.begin() + (off - start_pos);
|
||||
return getPosition();
|
||||
}
|
||||
else
|
||||
start_pos = off + iv_offset;
|
||||
start_pos = off;
|
||||
}
|
||||
else
|
||||
throw Exception("ReadEncryptedBuffer::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
|
||||
|
||||
initialize();
|
||||
return start_pos - iv_offset;
|
||||
return start_pos;
|
||||
}
|
||||
|
||||
off_t getPosition() override { return start_pos + offset() - iv_offset; }
|
||||
off_t getPosition() override { return start_pos + offset(); }
|
||||
|
||||
std::string getFileName() const override { return in->getFileName(); }
|
||||
|
||||
@ -83,7 +80,7 @@ private:
|
||||
|
||||
void initialize()
|
||||
{
|
||||
size_t in_pos = start_pos;
|
||||
size_t in_pos = start_pos + iv_offset;
|
||||
|
||||
String data;
|
||||
data.resize(buf_size);
|
||||
@ -101,7 +98,7 @@ private:
|
||||
data.resize(data_size);
|
||||
working_buffer.resize(data_size);
|
||||
|
||||
decryptor.Decrypt(data.data(), working_buffer.begin(), data_size, start_pos - iv_offset);
|
||||
decryptor.Decrypt(data.data(), working_buffer.begin(), data_size, start_pos);
|
||||
|
||||
pos = working_buffer.begin();
|
||||
initialized = true;
|
||||
@ -112,6 +109,8 @@ private:
|
||||
|
||||
Decryptor decryptor;
|
||||
bool initialized = false;
|
||||
|
||||
// current working_buffer.begin() offset from decrypted file
|
||||
size_t start_pos = 0;
|
||||
size_t iv_offset = 0;
|
||||
};
|
||||
|
@ -3,7 +3,6 @@
|
||||
#include <IO/WriteBufferFromFileBase.h>
|
||||
#include <Functions/FileEncryption.h>
|
||||
|
||||
#include <common/logger_useful.h>
|
||||
|
||||
namespace DB
|
||||
{
|
||||
@ -63,7 +62,7 @@ private:
|
||||
return;
|
||||
if (flush_iv)
|
||||
{
|
||||
writeText(iv.data(), *out);
|
||||
WriteIV(iv, *out);
|
||||
flush_iv = false;
|
||||
}
|
||||
|
||||
|
Loading…
Reference in New Issue
Block a user