diff --git a/src/Disks/DiskEncrypted.cpp b/src/Disks/DiskEncrypted.cpp index 6b456f06983..01fc71af084 100644 --- a/src/Disks/DiskEncrypted.cpp +++ b/src/Disks/DiskEncrypted.cpp @@ -62,15 +62,13 @@ std::unique_ptr DiskEncrypted::readFile( size_t mmap_threshold, MMappedFileCache * mmap_cache) const { - using namespace FileEncryption; - auto wrapped_path = wrappedPath(path); auto buffer = delegate->readFile(wrapped_path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache); auto iv = GetRandomString(kIVSize); size_t offset = 0; - if (delegate->getFileSize(wrapped_path)) + if (exists(path) && getFileSize(path)) { iv = ReadIV(kIVSize, *buffer); offset = kIVSize; @@ -84,16 +82,14 @@ std::unique_ptr DiskEncrypted::writeFile( size_t buf_size, WriteMode mode) { - using namespace FileEncryption; - auto wrapped_path = wrappedPath(path); auto iv = GetRandomString(kIVSize); - try { + if (exists(path) && getFileSize(path)) + { auto read_buffer = delegate->readFile(wrapped_path, kIVSize); iv = ReadIV(kIVSize, *read_buffer); } - catch ( ... ) { } auto buffer = delegate->writeFile(wrapped_path, buf_size, mode); return std::make_unique(buf_size, std::move(buffer), iv, key, @@ -161,6 +157,9 @@ void registerDiskEncrypted(DiskFactory & factory) String key = config.getString(config_prefix + ".key", ""); if (key.empty()) throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); + if (key.size() != CipherKeyLength(DefaultCipher())) + throw Exception("Expected key with size " + std::to_string(CipherKeyLength(DefaultCipher())) + ", got key with size " + std::to_string(key.size()), + ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG); auto wrapped_disk = map.find(wrapped_disk_name); if (wrapped_disk == map.end()) diff --git a/src/Functions/FileEncryption.h b/src/Functions/FileEncryption.h index e0264a937da..75bb17edfd8 100644 --- a/src/Functions/FileEncryption.h +++ b/src/Functions/FileEncryption.h @@ -1,7 +1,5 @@ #pragma once -#include - #include #include @@ -9,8 +7,6 @@ #include #include -#include - #include #include @@ -19,10 +15,11 @@ #include + namespace FileEncryption { -constexpr size_t kIVSize = 128 / CHAR_BIT / 2; +constexpr size_t kIVSize = sizeof(DB::UInt64); class InitVector { @@ -59,7 +56,6 @@ private: mutable String modify; }; - class EncryptionKey { public: @@ -87,6 +83,12 @@ String GetRandomString(size_t size) return iv; } +void WriteIV(const String & iv, DB::WriteBuffer & out) +{ + for (auto i : iv) + DB::writeChar(i, out); +} + size_t CipherKeyLength(const EVP_CIPHER * evp_cipher) { return static_cast(EVP_CIPHER_key_length(evp_cipher)); @@ -97,12 +99,16 @@ size_t CipherIVLength(const EVP_CIPHER * evp_cipher) return static_cast(EVP_CIPHER_iv_length(evp_cipher)); } +const EVP_CIPHER * DefaultCipher() +{ + return EVP_aes_128_ctr(); +} class Encryption { public: Encryption(const String & iv_, const EncryptionKey & key_, size_t offset_ = 0) - : evp_cipher(EVP_aes_128_ctr()) + : evp_cipher(DefaultCipher()) , init_vector(iv_) , key(key_) , block_size(CipherIVLength(evp_cipher)) @@ -127,13 +133,12 @@ protected: const EVP_CIPHER * get() const { return evp_cipher; } const EVP_CIPHER * evp_cipher; - const InitVector init_vector; - EncryptionKey key; + const String init_vector; + const EncryptionKey key; size_t block_size; - size_t offset = 0; // global offset + size_t offset = 0; }; - class Encryptor : public Encryption { public: @@ -145,22 +150,24 @@ public: if (!size) return; - auto iv = init_vector; + auto iv = InitVector(init_vector); iv.SetCounter(Blocks(offset)); - size_t first_block_size = FirstBlockSize(size, offset); + size_t first_block_size = FirstBlockSize(size, offset); if (offset) { buf.write(EncryptPartialBlock(plaintext, first_block_size, iv, offset).data(), first_block_size); + offset = BlockOffset(offset + first_block_size); size -= first_block_size; - offset += first_block_size; iv.Inc(); } if (size) { + size_t blocks = Blocks(size); buf.write(EncryptNBytes(plaintext + first_block_size, size, iv).data(), size); - offset += size; + iv.Inc(blocks); + offset = size - blocks * block_size; } } @@ -169,7 +176,7 @@ private: { using namespace OpenSSLDetails; if (size > block_size) - onError("Expecter partial block, got block with size > block_size: size = " + std::to_string(size)); + onError("Expecter partial block, got block with size > block_size: size = " + std::to_string(size) + " and offset = " + std::to_string(off)); String plaintext(block_size, '\0'); for (size_t i = 0; i < size; ++i) @@ -204,9 +211,6 @@ private: ciphertext_ref += output_len; - - //* always do nothing for default cipher - int final_output_len = 0; if (EVP_EncryptFinal_ex(evp_ctx, reinterpret_cast(ciphertext_ref), &final_output_len) != 1) @@ -230,21 +234,24 @@ public: if (!size) return; - auto iv = init_vector; - iv.SetCounter(Blocks(off)); - off = BlockOffset(off); + auto iv = InitVector(init_vector); + iv.SetCounter(Blocks(off)); + off = BlockOffset(off); size_t first_block_size = FirstBlockSize(size, off); if (off) { DecryptPartialBlock(buf, ciphertext, first_block_size, iv, off); size -= first_block_size; - iv.Inc(); + if (first_block_size + off == block_size) + iv.Inc(); } if (size) { + size_t blocks = Blocks(size); DecryptNBytes(buf, ciphertext + first_block_size, size, iv); + iv.Inc(blocks); } } @@ -275,7 +282,7 @@ private: auto * evp_ctx = evp_ctx_ptr.get(); if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1) - onError("Failed to initialize decryption context with cipher"); + onError("Failed to initialize encryption context with cipher"); if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr, reinterpret_cast(key.GetRef()), @@ -290,10 +297,7 @@ private: to += output_len; - - //* always do nothing for default cipher - - int final_output_len = 0; + int final_output_len = 0; if (EVP_DecryptFinal_ex(evp_ctx, reinterpret_cast(to), &final_output_len) != 1) onError("Failed to fetch ciphertext"); diff --git a/src/IO/ReadEncryptedBuffer.h b/src/IO/ReadEncryptedBuffer.h index 8bedbc3fc1b..90e0f676bf3 100644 --- a/src/IO/ReadEncryptedBuffer.h +++ b/src/IO/ReadEncryptedBuffer.h @@ -4,7 +4,6 @@ #include #include -#include namespace DB { @@ -24,7 +23,6 @@ public: , in(std::move(in_)) , buf_size(buf_size_) , decryptor(Decryptor(init_vector_, key_)) - , start_pos(iv_offset_) , iv_offset(iv_offset_) { } @@ -35,36 +33,35 @@ public: if (off < 0 && -off > getPosition()) throw Exception("SEEK_CUR shift out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND); - if (!working_buffer.empty() && size_t(offset() + off) < working_buffer.size()) + if (!working_buffer.empty() && static_cast(offset() + off) < working_buffer.size()) { pos += off; return getPosition(); } else - start_pos = off + getPosition() + iv_offset; + start_pos = off + getPosition(); } else if (whence == SEEK_SET) { if (off < 0) - throw Exception("SEEK_SET underflow", ErrorCodes::ARGUMENT_OUT_OF_BOUND); + throw Exception("SEEK_SET underflow: off = " + std::to_string(off), ErrorCodes::ARGUMENT_OUT_OF_BOUND); - if (!working_buffer.empty() && size_t(off) >= start_pos - working_buffer.size() - && size_t(off) < start_pos) + if (!working_buffer.empty() && static_cast(off) >= start_pos && static_cast(off) < (start_pos + working_buffer.size())) { - pos = working_buffer.end() - (start_pos - off); + pos = working_buffer.begin() + (off - start_pos); return getPosition(); } else - start_pos = off + iv_offset; + start_pos = off; } else throw Exception("ReadEncryptedBuffer::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND); initialize(); - return start_pos - iv_offset; + return start_pos; } - off_t getPosition() override { return start_pos + offset() - iv_offset; } + off_t getPosition() override { return start_pos + offset(); } std::string getFileName() const override { return in->getFileName(); } @@ -83,7 +80,7 @@ private: void initialize() { - size_t in_pos = start_pos; + size_t in_pos = start_pos + iv_offset; String data; data.resize(buf_size); @@ -101,7 +98,7 @@ private: data.resize(data_size); working_buffer.resize(data_size); - decryptor.Decrypt(data.data(), working_buffer.begin(), data_size, start_pos - iv_offset); + decryptor.Decrypt(data.data(), working_buffer.begin(), data_size, start_pos); pos = working_buffer.begin(); initialized = true; @@ -112,6 +109,8 @@ private: Decryptor decryptor; bool initialized = false; + + // current working_buffer.begin() offset from decrypted file size_t start_pos = 0; size_t iv_offset = 0; }; diff --git a/src/IO/WriteEncryptedBuffer.h b/src/IO/WriteEncryptedBuffer.h index c6180041ce0..c8c8c71f8f1 100644 --- a/src/IO/WriteEncryptedBuffer.h +++ b/src/IO/WriteEncryptedBuffer.h @@ -3,7 +3,6 @@ #include #include -#include namespace DB { @@ -63,7 +62,7 @@ private: return; if (flush_iv) { - writeText(iv.data(), *out); + WriteIV(iv, *out); flush_iv = false; }