implement read and write buffers (with debug=warning logs)

src/Functions/FileEncryption.h

@@ -0,0 +1,135 @@
+#pragma once
+
+#include <openssl/evp.h>
+#include <openssl/engine.h>
+
+#include <IO/ReadBuffer.h>
+#include <IO/WriteBuffer.h>
+
+#include <IO/WriteHelpers.h>
+
+#include <common/logger_useful.h>
+#include <sys/random.h>
+
+namespace DB
+{
+
+class InitVector
+{
+public:
+    InitVector(String iv_) : iv(iv_) { }
+
+    size_t Size() const { return iv.size(); }
+    String Data() const { return iv; }
+
+private:
+    String iv;
+};
+
+class EncryptionKey
+{
+public:
+    EncryptionKey(String key_) : key(key_) { }
+    String Get() const { return key; }
+
+private:
+    String key;
+};
+
+
+InitVector ReadIV(size_t size, ReadBuffer & in)
+{
+    Poco::Logger * log = &Poco::Logger::get("FileEncryption");
+    LOG_WARNING(log, "ReadInitVector {}", size);
+    String iv;
+    iv.resize(size);
+    in.readStrict(reinterpret_cast<char *>(iv.data()), size);
+    LOG_WARNING(log, "read iv = {}", iv);
+    return InitVector(iv);
+}
+
+InitVector GetRandomIV(size_t size)
+{
+    Poco::Logger * log = &Poco::Logger::get("FileEncryption");
+    LOG_WARNING(log, "GetRandomIV {}", size);
+    String iv;
+    iv.resize(size);
+    getrandom(iv.data(), size, GRND_NONBLOCK);
+    LOG_WARNING(log, "generated iv = {}", iv);
+    return InitVector(iv);
+}
+
+void WriteIV(const InitVector & iv, WriteBuffer & out)
+{
+    Poco::Logger * log = &Poco::Logger::get("FileEncryption");
+    LOG_WARNING(log, "WriteInitVector {}", iv.Data());
+    writeText(iv.Data(), out);
+}
+
+class Encryption
+{
+public:
+    Encryption(const InitVector & iv_, const EncryptionKey & key_, const EVP_CIPHER * evp_cipher_, size_t offset_ = 0)
+        : evp_cipher(evp_cipher_)
+        , iv(iv_)
+        , key(key_)
+        , block_size(static_cast<size_t>(EVP_CIPHER_block_size(evp_cipher_)))
+    {
+        blocks = Blocks(offset_);
+        offset = BlockOffset(offset_);
+    }
+
+    size_t SizeByInputSize(size_t input_size) const { return input_size; }
+    Poco::Logger * log = &Poco::Logger::get("FileEncryption");
+
+private:
+
+    size_t Blocks(size_t pos) { return pos / block_size; }
+
+    size_t BlockOffset(size_t pos) const { return pos % block_size; }
+
+//    size_t BlocksAlign(size_t pos) const { return pos - BlockOffset(pos); }
+
+//    size_t BlockStartPos(size_t pos) const { return iv.Size() + pos - BlockOffset(pos); }
+
+    const EVP_CIPHER * get() const { return evp_cipher; }
+
+    const EVP_CIPHER * evp_cipher;
+    InitVector iv;
+    EncryptionKey key;
+    size_t blocks = 0;
+    size_t block_size;
+    size_t offset = 0;
+};
+
+class Encryptor : public Encryption
+{
+public:
+    Encryptor(const InitVector & iv_, const EncryptionKey & key_, const EVP_CIPHER * evp_cipher_, size_t offset_)
+        : Encryption(iv_, key_, evp_cipher_, offset_) { }
+
+    void Encrypt(const char * plaintext, WriteBuffer & buf, size_t size)
+    {
+        if (!size)
+            return;
+        LOG_WARNING(log, "Encrypt with size {}", size);
+        buf.write(plaintext, size);
+    }
+};
+
+class Decryptor : public Encryption
+{
+public:
+    Decryptor(const InitVector & iv_, const EncryptionKey & key_, const EVP_CIPHER * evp_cipher_)
+        : Encryption(iv_, key_, evp_cipher_) { }
+
+    void Decrypt(const char * ciphertext, const BufferBase::Buffer & buf, size_t size)
+    {
+        if (!size)
+            return;
+        LOG_WARNING(log, "Decrypt with size {}", size);
+        WriteBuffer(buf.begin(), buf.size()).write(ciphertext, size);
+    }
+};
+
+}

src/IO/ReadEncryptedBuffer.h

@@ -0,0 +1,131 @@
+#pragma once
+
+#include <IO/ReadBufferFromFileBase.h>
+#include <Functions/FileEncryption.h>
+
+#include <common/logger_useful.h>
+
+namespace DB
+{
+
+class ReadEncryptedBuffer : public ReadBufferFromFileBase
+{
+public:
+    explicit ReadEncryptedBuffer(
+        size_t buf_size_,
+        std::unique_ptr<ReadBufferFromFileBase> in_,
+        const EVP_CIPHER * evp_cipher_,
+        const InitVector & init_vector_,
+        const EncryptionKey & key_,
+	const size_t iv_offset_)
+        : ReadBufferFromFileBase(buf_size_, nullptr, 0)
+        , in(std::move(in_))
+        , buf_size(buf_size_)
+        , iv(init_vector_)
+        , decryptor(Decryptor(init_vector_, key_, evp_cipher_))
+        , start_pos(iv_offset_)
+	, iv_offset(iv_offset_)
+    {
+        LOG_WARNING(log, "ReadEncryptedBuffer() buf_size = {}; iv = {}\n", buf_size, iv.Data());
+    }
+
+    off_t seek(off_t off, int whence) override
+    {
+        LOG_WARNING(log, "ReadEncryptedBuffer::seek()");
+        if (whence == SEEK_CUR)
+        {
+            if (off < 0 && -off > getPosition())
+                throw Exception("SEEK_CUR shift out of bounds", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+            if (!working_buffer.empty() && size_t(offset() + off) < working_buffer.size())
+            {
+                pos += off;
+                return getPosition();
+            }
+            else
+            {
+                start_pos = off + getPosition() + iv_offset;
+            }
+        }
+        else if (whence == SEEK_SET)
+        {
+            if (off < 0)
+                throw Exception("SEEK_SET underflow", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+            if (!working_buffer.empty() && size_t(off) >= start_pos - working_buffer.size()
+                && size_t(off) < start_pos)
+            {
+                pos = working_buffer.end() - (start_pos - off);
+                return getPosition();
+            }
+            else
+            {
+                start_pos = off + iv_offset;
+            }
+        }
+        else
+            throw Exception("ReadEncryptedBuffer::seek expects SEEK_SET or SEEK_CUR as whence", ErrorCodes::ARGUMENT_OUT_OF_BOUND);
+
+        initialize();
+        return start_pos - iv_offset;
+    }
+
+    off_t getPosition() override { return bytes + offset(); }
+
+    std::string getFileName() const override { return in->getFileName(); }
+
+private:
+
+    bool nextImpl() override
+    {
+        LOG_WARNING(log, "ReadEncryptedBuffer::nextImpl()\n");
+        if (in->eof())
+            return false;
+
+        if (initialized)
+            start_pos += working_buffer.size();
+        initialize();
+        return true;
+    }
+
+    void initialize()
+    {
+        LOG_WARNING(log, "ReadEncryptedBuffer::initialize()\n");
+        size_t in_pos = start_pos;
+
+        String data;
+	data.resize(buf_size);
+        size_t data_size = 0;
+
+        LOG_WARNING(log, "in_pos = {}, expected_size = {}\n", in_pos, buf_size);
+        in->seek(in_pos, SEEK_SET);
+        while (data_size < buf_size && !in->eof())
+        {
+            auto size = in->read(data.data() + data_size, buf_size - data_size);
+            data_size += size;
+            in_pos += size;
+            in->seek(in_pos, SEEK_SET);
+        }
+
+        data.resize(data_size);
+	working_buffer.resize(data_size);
+
+        LOG_WARNING(log, "read {} bytes : {}\n", data_size, data);
+        decryptor.Decrypt(data.data(), working_buffer, data_size);
+
+	pos = working_buffer.begin();
+        initialized = true;
+    }
+
+    std::unique_ptr<ReadBufferFromFileBase> in;
+    size_t buf_size;
+
+    InitVector iv;
+    Decryptor decryptor;
+    bool initialized = false;
+    size_t start_pos = 0;
+    size_t iv_offset = 0;
+    Poco::Logger * log = &Poco::Logger::get("ReadEncryptedBuffer");
+};
+
+}

src/IO/WriteEncryptedBuffer.h

@@ -0,0 +1,88 @@
+#pragma once
+
+#include <IO/WriteBufferFromFileBase.h>
+#include <Functions/FileEncryption.h>
+
+#include <common/logger_useful.h>
+
+namespace DB
+{
+
+class WriteEncryptedBuffer : public WriteBufferFromFileBase
+{
+public:
+    WriteEncryptedBuffer(
+        size_t buf_size_,
+        std::unique_ptr<WriteBufferFromFileBase> out_,
+        const EVP_CIPHER * evp_cipher_,
+        const InitVector & init_vector_,
+        const EncryptionKey & key_,
+        const size_t & file_size)
+        : WriteBufferFromFileBase(buf_size_, nullptr, 0)
+        , out(std::move(out_))
+        , flush_iv(!file_size)
+        , iv(init_vector_)
+        , encryptor(Encryptor(init_vector_, key_, evp_cipher_, file_size))
+    { }
+
+    ~WriteEncryptedBuffer() override
+    {
+        LOG_WARNING(log, "~WriteEncryptedBuffer()");
+        try
+        {
+            finalize();
+	}
+	catch (...)
+	{
+            tryLogCurrentException(__PRETTY_FUNCTION__);
+	}
+    }
+
+    void finalize() override
+    {
+        LOG_WARNING(log, "WriteEncryptedBuffer::finalize()");
+        if (finalized)
+	    return;
+
+        LOG_WARNING(log, "WriteEncryptedBuffer::next()");
+        next();
+        LOG_WARNING(log, "WriteEncryptedBuffer::out::finalize()");
+        out->finalize();
+
+	finalized = true;
+    }
+
+    void sync() override
+    {
+        LOG_WARNING(log, "WriteEncryptedBuffer::sync()");
+        out->sync();
+    }
+
+    std::string getFileName() const override { return out->getFileName(); }
+
+private:
+    void nextImpl() override
+    {
+        LOG_WARNING(log, "WriteEncryptedBuffer::nextImpl()");
+        if (!offset())
+            return;
+        if (flush_iv)
+        {
+            LOG_WARNING(log, "flush_iv == true");
+            WriteIV(iv, *out);
+            flush_iv = false;
+        }
+
+        encryptor.Encrypt(working_buffer.begin(), *out, offset());
+    }
+
+    bool finalized = false;
+    std::unique_ptr<WriteBufferFromFileBase> out;
+
+    bool flush_iv;
+    InitVector iv;
+    Encryptor encryptor;
+    Poco::Logger * log = &Poco::Logger::get("WriteEncryptedBuffer");
+};
+
+}