thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-09-29 21:20:49 +00:00
Code Issues Packages Projects Releases Wiki Activity

remove debug information, fix seek issue and encryption issue

Browse Source
This commit is contained in:
Alexandra 2021-05-17 16:15:21 +00:00
parent 10a5b734ee
commit af420569f8
4 changed files with 51 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
src/Disks/DiskEncrypted.cpp
View File

@ -62,15 +62,13 @@ std::unique_ptr<ReadBufferFromFileBase> DiskEncrypted::readFile(
size_t mmap_threshold,
MMappedFileCache * mmap_cache) const
{
using namespace FileEncryption;
auto wrapped_path = wrappedPath(path);
auto buffer = delegate->readFile(wrapped_path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
auto iv = GetRandomString(kIVSize);
size_t offset = 0;
if (delegate->getFileSize(wrapped_path))
if (exists(path) && getFileSize(path))
{
iv = ReadIV(kIVSize, *buffer);
offset = kIVSize;
@ -84,16 +82,14 @@ std::unique_ptr<WriteBufferFromFileBase> DiskEncrypted::writeFile(
size_t buf_size,
WriteMode mode)
{
using namespace FileEncryption;
auto wrapped_path = wrappedPath(path);
auto iv = GetRandomString(kIVSize);
try {
if (exists(path) && getFileSize(path))
{
auto read_buffer = delegate->readFile(wrapped_path, kIVSize);
iv = ReadIV(kIVSize, *read_buffer);
}
catch ( ... ) { }
auto buffer = delegate->writeFile(wrapped_path, buf_size, mode);
return std::make_unique<WriteEncryptedBuffer>(buf_size, std::move(buffer), iv, key,
@ -161,6 +157,9 @@ void registerDiskEncrypted(DiskFactory & factory)
String key = config.getString(config_prefix + ".key", "");
if (key.empty())
throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
if (key.size() != CipherKeyLength(DefaultCipher()))
throw Exception("Expected key with size " + std::to_string(CipherKeyLength(DefaultCipher())) + ", got key with size " + std::to_string(key.size()),
ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
auto wrapped_disk = map.find(wrapped_disk_name);
if (wrapped_disk == map.end())

60
src/Functions/FileEncryption.h
View File

@ -1,7 +1,5 @@
#pragma once
#include <fstream>
#include <openssl/evp.h>
#include <openssl/engine.h>
@ -9,8 +7,6 @@
#include <IO/WriteBuffer.h>
#include <IO/WriteHelpers.h>
#include <common/logger_useful.h>
#include <sys/random.h>
#include <limits>
@ -19,10 +15,11 @@
#include <Functions/FunctionsAES.h>
namespace FileEncryption
{
constexpr size_t kIVSize = 128 / CHAR_BIT / 2;
constexpr size_t kIVSize = sizeof(DB::UInt64);
class InitVector
{
@ -59,7 +56,6 @@ private:
mutable String modify;
};
class EncryptionKey
{
public:
@ -87,6 +83,12 @@ String GetRandomString(size_t size)
return iv;
}
void WriteIV(const String & iv, DB::WriteBuffer & out)
{
for (auto i : iv)
DB::writeChar(i, out);
}
size_t CipherKeyLength(const EVP_CIPHER * evp_cipher)
{
return static_cast<size_t>(EVP_CIPHER_key_length(evp_cipher));
@ -97,12 +99,16 @@ size_t CipherIVLength(const EVP_CIPHER * evp_cipher)
return static_cast<size_t>(EVP_CIPHER_iv_length(evp_cipher));
}
const EVP_CIPHER * DefaultCipher()
{
return EVP_aes_128_ctr();
}
class Encryption
{
public:
Encryption(const String & iv_, const EncryptionKey & key_, size_t offset_ = 0)
: evp_cipher(EVP_aes_128_ctr())
: evp_cipher(DefaultCipher())
, init_vector(iv_)
, key(key_)
, block_size(CipherIVLength(evp_cipher))
@ -127,13 +133,12 @@ protected:
const EVP_CIPHER * get() const { return evp_cipher; }
const EVP_CIPHER * evp_cipher;
const InitVector init_vector;
EncryptionKey key;
const String init_vector;
const EncryptionKey key;
size_t block_size;
size_t offset = 0; // global offset
size_t offset = 0;
};
class Encryptor : public Encryption
{
public:
@ -145,22 +150,24 @@ public:
if (!size)
return;
auto iv = init_vector;
auto iv = InitVector(init_vector);
iv.SetCounter(Blocks(offset));
size_t first_block_size = FirstBlockSize(size, offset);
size_t first_block_size = FirstBlockSize(size, offset);
if (offset)
{
buf.write(EncryptPartialBlock(plaintext, first_block_size, iv, offset).data(), first_block_size);
offset = BlockOffset(offset + first_block_size);
size -= first_block_size;
offset += first_block_size;
iv.Inc();
}
if (size)
{
size_t blocks = Blocks(size);
buf.write(EncryptNBytes(plaintext + first_block_size, size, iv).data(), size);
offset += size;
iv.Inc(blocks);
offset = size - blocks * block_size;
}
}
@ -169,7 +176,7 @@ private:
{
using namespace OpenSSLDetails;
if (size > block_size)
onError("Expecter partial block, got block with size > block_size: size = " + std::to_string(size));
onError("Expecter partial block, got block with size > block_size: size = " + std::to_string(size) + " and offset = " + std::to_string(off));
String plaintext(block_size, '\0');
for (size_t i = 0; i < size; ++i)
@ -204,9 +211,6 @@ private:
ciphertext_ref += output_len;
//* always do nothing for default cipher
int final_output_len = 0;
if (EVP_EncryptFinal_ex(evp_ctx,
reinterpret_cast<unsigned char*>(ciphertext_ref), &final_output_len) != 1)
@ -230,21 +234,24 @@ public:
if (!size)
return;
auto iv = init_vector;
iv.SetCounter(Blocks(off));
off = BlockOffset(off);
auto iv = InitVector(init_vector);
iv.SetCounter(Blocks(off));
off = BlockOffset(off);
size_t first_block_size = FirstBlockSize(size, off);
if (off)
{
DecryptPartialBlock(buf, ciphertext, first_block_size, iv, off);
size -= first_block_size;
iv.Inc();
if (first_block_size + off == block_size)
iv.Inc();
}
if (size)
{
size_t blocks = Blocks(size);
DecryptNBytes(buf, ciphertext + first_block_size, size, iv);
iv.Inc(blocks);
}
}
@ -275,7 +282,7 @@ private:
auto * evp_ctx = evp_ctx_ptr.get();
if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1)
onError("Failed to initialize decryption context with cipher");
onError("Failed to initialize encryption context with cipher");
if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr,
reinterpret_cast<const unsigned char*>(key.GetRef()),
@ -290,10 +297,7 @@ private:
to += output_len;
//* always do nothing for default cipher
int final_output_len = 0;
int final_output_len = 0;
if (EVP_DecryptFinal_ex(evp_ctx,
reinterpret_cast<unsigned char*>(to), &final_output_len) != 1)
onError("Failed to fetch ciphertext");

25
src/IO/ReadEncryptedBuffer.h
View File

@ -4,7 +4,6 @@
#include <IO/WriteBufferFromFileBase.h>
#include <Functions/FileEncryption.h>
#include <common/logger_useful.h>
namespace DB
{
@ -24,7 +23,6 @@ public:
, in(std::move(in_))
, buf_size(buf_size_)
, decryptor(Decryptor(init_vector_, key_))
, start_pos(iv_offset_)
, iv_offset(iv_offset_)
{ }
@ -35,36 +33,35 @@ public:
if (off < 0 && -off > getPosition())
throw Exception("SEEK_CUR shift out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
if (!working_buffer.empty() && size_t(offset() + off) < working_buffer.size())
if (!working_buffer.empty() && static_cast<size_t>(offset() + off) < working_buffer.size())
{
pos += off;
return getPosition();
}
else
start_pos = off + getPosition() + iv_offset;
start_pos = off + getPosition();
}
else if (whence == SEEK_SET)
{
if (off < 0)
throw Exception("SEEK_SET underflow", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
throw Exception("SEEK_SET underflow: off = " + std::to_string(off), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
if (!working_buffer.empty() && size_t(off) >= start_pos - working_buffer.size()
&& size_t(off) < start_pos)
if (!working_buffer.empty() && static_cast<size_t>(off) >= start_pos && static_cast<size_t>(off) < (start_pos + working_buffer.size()))
{
pos = working_buffer.end() - (start_pos - off);
pos = working_buffer.begin() + (off - start_pos);
return getPosition();
}
else
start_pos = off + iv_offset;
start_pos = off;
}
else
throw Exception("ReadEncryptedBuffer::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
initialize();
return start_pos - iv_offset;
return start_pos;
}
off_t getPosition() override { return start_pos + offset() - iv_offset; }
off_t getPosition() override { return start_pos + offset(); }
std::string getFileName() const override { return in->getFileName(); }
@ -83,7 +80,7 @@ private:
void initialize()
{
size_t in_pos = start_pos;
size_t in_pos = start_pos + iv_offset;
String data;
data.resize(buf_size);
@ -101,7 +98,7 @@ private:
data.resize(data_size);
working_buffer.resize(data_size);
decryptor.Decrypt(data.data(), working_buffer.begin(), data_size, start_pos - iv_offset);
decryptor.Decrypt(data.data(), working_buffer.begin(), data_size, start_pos);
pos = working_buffer.begin();
initialized = true;
@ -112,6 +109,8 @@ private:
Decryptor decryptor;
bool initialized = false;
// current working_buffer.begin() offset from decrypted file
size_t start_pos = 0;
size_t iv_offset = 0;
};

3
src/IO/WriteEncryptedBuffer.h
View File

@ -3,7 +3,6 @@
#include <IO/WriteBufferFromFileBase.h>
#include <Functions/FileEncryption.h>
#include <common/logger_useful.h>
namespace DB
{
@ -63,7 +62,7 @@ private:
return;
if (flush_iv)
{
writeText(iv.data(), *out);
WriteIV(iv, *out);
flush_iv = false;
}