remove debug information, fix seek issue and encryption issue

src/Disks/DiskEncrypted.cpp

@@ -62,15 +62,13 @@ std::unique_ptr<ReadBufferFromFileBase> DiskEncrypted::readFile(
     size_t mmap_threshold,
     MMappedFileCache * mmap_cache) const
 {
-    using namespace FileEncryption;
-
     auto wrapped_path = wrappedPath(path);
     auto buffer = delegate->readFile(wrapped_path, buf_size, estimated_size, aio_threshold, mmap_threshold, mmap_cache);
 
     auto iv = GetRandomString(kIVSize);
     size_t offset = 0;
 
-    if (delegate->getFileSize(wrapped_path))
+    if (exists(path) && getFileSize(path))
     {
         iv = ReadIV(kIVSize, *buffer);
         offset = kIVSize;
@@ -84,16 +82,14 @@ std::unique_ptr<WriteBufferFromFileBase> DiskEncrypted::writeFile(
     size_t buf_size,
     WriteMode mode)
 {
-    using namespace FileEncryption;
-
     auto wrapped_path = wrappedPath(path);
     auto iv = GetRandomString(kIVSize);
 
-    try {
+    if (exists(path) && getFileSize(path))
+    {
         auto read_buffer = delegate->readFile(wrapped_path, kIVSize);
         iv = ReadIV(kIVSize, *read_buffer);
     }
-    catch ( ... ) { }
 
     auto buffer = delegate->writeFile(wrapped_path, buf_size, mode);
     return std::make_unique<WriteEncryptedBuffer>(buf_size, std::move(buffer), iv, key,
@@ -161,6 +157,9 @@ void registerDiskEncrypted(DiskFactory & factory)
         String key = config.getString(config_prefix + ".key", "");
         if (key.empty())
             throw Exception("Encrypted disk key can not be empty. Disk " + name, ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
+	if (key.size() != CipherKeyLength(DefaultCipher()))
+            throw Exception("Expected key with size " + std::to_string(CipherKeyLength(DefaultCipher())) + ", got key with size " + std::to_string(key.size()),
+			    ErrorCodes::UNKNOWN_ELEMENT_IN_CONFIG);
 
         auto wrapped_disk = map.find(wrapped_disk_name);
         if (wrapped_disk == map.end())

src/Functions/FileEncryption.h

@@ -1,7 +1,5 @@
 #pragma once
 
-#include <fstream>
-
 #include <openssl/evp.h>
 #include <openssl/engine.h>
 
@@ -9,8 +7,6 @@
 #include <IO/WriteBuffer.h>
 #include <IO/WriteHelpers.h>
 
-#include <common/logger_useful.h>
-
 #include <sys/random.h>
 
 #include <limits>
@@ -19,10 +15,11 @@
 
 #include <Functions/FunctionsAES.h>
 
+
 namespace FileEncryption
 {
 
-constexpr size_t kIVSize = 128 / CHAR_BIT / 2;
+constexpr size_t kIVSize = sizeof(DB::UInt64);
 
 class InitVector
 {
@@ -59,7 +56,6 @@ private:
     mutable String modify;
 };
 
-
 class EncryptionKey
 {
 public:
@@ -87,6 +83,12 @@ String GetRandomString(size_t size)
     return iv;
 }
 
+void WriteIV(const String & iv, DB::WriteBuffer & out)
+{
+    for (auto i : iv)
+	DB::writeChar(i, out);
+}
+
 size_t CipherKeyLength(const EVP_CIPHER * evp_cipher)
 {
     return static_cast<size_t>(EVP_CIPHER_key_length(evp_cipher));
@@ -97,12 +99,16 @@ size_t CipherIVLength(const EVP_CIPHER * evp_cipher)
     return static_cast<size_t>(EVP_CIPHER_iv_length(evp_cipher));
 }
 
+const EVP_CIPHER * DefaultCipher()
+{
+    return EVP_aes_128_ctr();
+}
 
 class Encryption
 {
 public:
     Encryption(const String & iv_, const EncryptionKey & key_, size_t offset_ = 0)
-        : evp_cipher(EVP_aes_128_ctr())
+        : evp_cipher(DefaultCipher())
         , init_vector(iv_)
         , key(key_)
         , block_size(CipherIVLength(evp_cipher))
@@ -127,13 +133,12 @@ protected:
     const EVP_CIPHER * get() const { return evp_cipher; }
 
     const EVP_CIPHER * evp_cipher;
-    const InitVector init_vector;
+    const String init_vector;
-    EncryptionKey key;
+    const EncryptionKey key;
     size_t block_size;
-    size_t offset = 0; // global offset
+    size_t offset = 0;
 };
 
-
 class Encryptor : public Encryption
 {
 public:
@@ -145,22 +150,24 @@ public:
         if (!size)
             return;
 
-        auto iv = init_vector;
+        auto iv = InitVector(init_vector);
 	iv.SetCounter(Blocks(offset));
-        size_t first_block_size = FirstBlockSize(size, offset);
 
+        size_t first_block_size = FirstBlockSize(size, offset);
         if (offset)
         {
             buf.write(EncryptPartialBlock(plaintext, first_block_size, iv, offset).data(), first_block_size);
+            offset = BlockOffset(offset + first_block_size);
             size -= first_block_size;
-	    offset += first_block_size;
             iv.Inc();
         }
 
         if (size)
         {
+            size_t blocks = Blocks(size);
             buf.write(EncryptNBytes(plaintext + first_block_size, size, iv).data(), size);
-	    offset += size;
+            iv.Inc(blocks);
+            offset = size - blocks * block_size;
         }
     }
 
@@ -169,7 +176,7 @@ private:
     {
         using namespace OpenSSLDetails;
         if (size > block_size)
-            onError("Expecter partial block, got block with size > block_size: size = " + std::to_string(size));
+            onError("Expecter partial block, got block with size > block_size: size = " + std::to_string(size) + " and offset = " + std::to_string(off));
 
         String plaintext(block_size, '\0');
         for (size_t i = 0; i < size; ++i)
@@ -204,9 +211,6 @@ private:
 
         ciphertext_ref += output_len;
 
-
-	//* always do nothing for default cipher
-
         int final_output_len = 0;
         if (EVP_EncryptFinal_ex(evp_ctx,
             reinterpret_cast<unsigned char*>(ciphertext_ref), &final_output_len) != 1)
@@ -230,7 +234,7 @@ public:
         if (!size)
             return;
 
-        auto iv = init_vector;
+        auto iv = InitVector(init_vector);
         iv.SetCounter(Blocks(off));
         off = BlockOffset(off);
 
@@ -239,12 +243,15 @@ public:
         {
             DecryptPartialBlock(buf, ciphertext, first_block_size, iv, off);
             size -= first_block_size;
+            if (first_block_size + off == block_size)
                 iv.Inc();
         }
 
         if (size)
         {
+            size_t blocks = Blocks(size);
             DecryptNBytes(buf, ciphertext + first_block_size, size, iv);
+            iv.Inc(blocks);
         }
     }
 
@@ -275,7 +282,7 @@ private:
         auto * evp_ctx = evp_ctx_ptr.get();
 
         if (EVP_DecryptInit_ex(evp_ctx, evp_cipher, nullptr, nullptr, nullptr) != 1)
-            onError("Failed to initialize decryption context with cipher");
+            onError("Failed to initialize encryption context with cipher");
 
         if (EVP_DecryptInit_ex(evp_ctx, nullptr, nullptr,
             reinterpret_cast<const unsigned char*>(key.GetRef()),
@@ -290,9 +297,6 @@ private:
 
         to += output_len;
 
-
-	//* always do nothing for default cipher
-
         int final_output_len = 0;
         if (EVP_DecryptFinal_ex(evp_ctx,
             reinterpret_cast<unsigned char*>(to), &final_output_len) != 1)

src/IO/ReadEncryptedBuffer.h

@@ -4,7 +4,6 @@
 #include <IO/WriteBufferFromFileBase.h>
 #include <Functions/FileEncryption.h>
 
-#include <common/logger_useful.h>
 
 namespace DB
 {
@@ -24,7 +23,6 @@ public:
         , in(std::move(in_))
         , buf_size(buf_size_)
         , decryptor(Decryptor(init_vector_, key_))
-        , start_pos(iv_offset_)
 	, iv_offset(iv_offset_)
     { }
 
@@ -35,36 +33,35 @@ public:
             if (off < 0 && -off > getPosition())
                 throw Exception("SEEK_CUR shift out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
 
-            if (!working_buffer.empty() && size_t(offset() + off) < working_buffer.size())
+            if (!working_buffer.empty() && static_cast<size_t>(offset() + off) < working_buffer.size())
             {
                 pos += off;
                 return getPosition();
             }
             else
-                start_pos = off + getPosition() + iv_offset;
+                start_pos = off + getPosition();
         }
         else if (whence == SEEK_SET)
         {
             if (off < 0)
-                throw Exception("SEEK_SET underflow", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+                throw Exception("SEEK_SET underflow: off = " + std::to_string(off), ErrorCodes::ARGUMENT_OUT_OF_BOUND);
 
-            if (!working_buffer.empty() && size_t(off) >= start_pos - working_buffer.size()
+            if (!working_buffer.empty() && static_cast<size_t>(off) >= start_pos && static_cast<size_t>(off) < (start_pos + working_buffer.size()))
-                && size_t(off) < start_pos)
             {
-                pos = working_buffer.end() - (start_pos - off);
+                pos = working_buffer.begin() + (off - start_pos);
                 return getPosition();
             }
             else
-                start_pos = off + iv_offset;
+                start_pos = off;
         }
         else
             throw Exception("ReadEncryptedBuffer::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
 
         initialize();
-        return start_pos - iv_offset;
+        return start_pos;
     }
 
-    off_t getPosition() override { return start_pos + offset() - iv_offset; }
+    off_t getPosition() override { return start_pos + offset(); }
 
     std::string getFileName() const override { return in->getFileName(); }
 
@@ -83,7 +80,7 @@ private:
 
     void initialize()
     {
-        size_t in_pos = start_pos;
+        size_t in_pos = start_pos + iv_offset;
 
         String data;
 	data.resize(buf_size);
@@ -101,7 +98,7 @@ private:
         data.resize(data_size);
 	working_buffer.resize(data_size);
 
-        decryptor.Decrypt(data.data(), working_buffer.begin(), data_size, start_pos - iv_offset);
+        decryptor.Decrypt(data.data(), working_buffer.begin(), data_size, start_pos);
 
 	pos = working_buffer.begin();
         initialized = true;
@@ -112,6 +109,8 @@ private:
 
     Decryptor decryptor;
     bool initialized = false;
+
+    // current working_buffer.begin() offset from decrypted file
     size_t start_pos = 0;
     size_t iv_offset = 0;
 };

src/IO/WriteEncryptedBuffer.h

@@ -3,7 +3,6 @@
 #include <IO/WriteBufferFromFileBase.h>
 #include <Functions/FileEncryption.h>
 
-#include <common/logger_useful.h>
 
 namespace DB
 {
@@ -63,7 +62,7 @@ private:
             return;
         if (flush_iv)
         {
-            writeText(iv.data(), *out);
+            WriteIV(iv, *out);
             flush_iv = false;
         }