ClickHouse/dbms/src/Storages/IStorage.h

#pragma once

#include <Core/Names.h>
#include <Common/Exception.h>
#include <Core/QueryProcessingStage.h>
#include <Storages/ITableDeclaration.h>
#include <Poco/RWLock.h>
#include <memory>
#include <experimental/optional>


namespace DB
{

namespace ErrorCodes
{
    extern const int TABLE_IS_DROPPED;
}

class Context;
class IBlockInputStream;
class IBlockOutputStream;

using BlockOutputStreamPtr = std::shared_ptr<IBlockOutputStream>;
using BlockInputStreamPtr = std::shared_ptr<IBlockInputStream>;
using BlockInputStreams = std::vector<BlockInputStreamPtr>;


class IStorage;

using StoragePtr = std::shared_ptr<IStorage>;

class IAST;

using ASTPtr = std::shared_ptr<IAST>;

struct Settings;

class AlterCommands;

/// For RESHARD PARTITION.
using WeightedZooKeeperPath = std::pair<String, UInt64>;
using WeightedZooKeeperPaths = std::vector<WeightedZooKeeperPath>;



/** Does not allow changing the table description (including rename and delete the table).
  * If during any operation the table structure should remain unchanged, you need to hold such a lock for all of its time.
  * For example, you need to hold such a lock for the duration of the entire SELECT or INSERT query and for the whole time the merge of the set of parts
  *  (but between the selection of parts for the merge and their merging, the table structure can change).
  * NOTE: This is a lock to "read" the table's description. To change the table description, you need to take the TableStructureWriteLock.
  */
class TableStructureReadLock
{
private:
    friend class IStorage;

    StoragePtr storage;
    /// Order is important.
    std::experimental::optional<Poco::ScopedReadRWLock> data_lock;
    std::experimental::optional<Poco::ScopedReadRWLock> structure_lock;

public:
    TableStructureReadLock(StoragePtr storage_, bool lock_structure, bool lock_data);
};

using TableStructureReadLockPtr = std::shared_ptr<TableStructureReadLock>;
using TableStructureReadLocks = std::vector<TableStructureReadLockPtr>;

using TableStructureWriteLockPtr = std::unique_ptr<Poco::ScopedWriteRWLock>;
using TableDataWriteLockPtr = std::unique_ptr<Poco::ScopedWriteRWLock>;
using TableFullWriteLockPtr = std::pair<TableDataWriteLockPtr, TableStructureWriteLockPtr>;


/** Storage. Responsible for
  * - storage of the table data;
  * - the definition in which file (or not the file) the data is stored;
  * - search for data and update data;
  * - data storage structure (compression, etc.)
  * - concurrent access to data (locks, etc.)
  */
class IStorage : public std::enable_shared_from_this<IStorage>, private boost::noncopyable, public ITableDeclaration
{
public:
    /// The main name of the table type (for example, StorageMergeTree).
    virtual std::string getName() const = 0;

    /** Returns true if the store receives data from a remote server or servers. */
    virtual bool isRemote() const { return false; }

    /** Returns true if the storage supports queries with the SAMPLE section. */
    virtual bool supportsSampling() const { return false; }

    /** Returns true if the storage supports queries with the FINAL section. */
    virtual bool supportsFinal() const { return false; }

    /** Returns true if the storage supports queries with the PREWHERE section. */
    virtual bool supportsPrewhere() const { return false; }

    /** Returns true if the storage supports multiple replicas. */
    virtual bool supportsParallelReplicas() const { return false; }

    /** Does not allow you to change the structure or name of the table.
      * If you change the data in the table, you will need to specify will_modify_data = true.
      * This will take an extra lock that does not allow starting ALTER MODIFY.
      *
      * WARNING: You need to call methods from ITableDeclaration under such a lock. Without it, they are not thread safe.
      * WARNING: To avoid deadlocks, this method must not be called under lock of Context.
      */
    TableStructureReadLockPtr lockStructure(bool will_modify_data)
    {
        TableStructureReadLockPtr res = std::make_shared<TableStructureReadLock>(shared_from_this(), true, will_modify_data);
        if (is_dropped)
            throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED);
        return res;
    }

    /** Does not allow reading the table structure. It is taken for ALTER, RENAME and DROP.
      */
    TableFullWriteLockPtr lockForAlter()
    {
        /// The calculation order is important.
        auto data_lock = lockDataForAlter();
        auto structure_lock = lockStructureForAlter();

        return {std::move(data_lock), std::move(structure_lock)};
    }

    /** Does not allow changing the data in the table. (Moreover, does not give a look at the structure of the table with the intention to change the data).
      * It is taken during write temporary data in ALTER MODIFY.
      * Under this lock, you can take lockStructureForAlter() to change the structure of the table.
      */
    TableDataWriteLockPtr lockDataForAlter()
    {
        auto res = std::make_unique<Poco::ScopedWriteRWLock>(data_lock);
        if (is_dropped)
            throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED);
        return res;
    }

    TableStructureWriteLockPtr lockStructureForAlter()
    {
        auto res = std::make_unique<Poco::ScopedWriteRWLock>(structure_lock);
        if (is_dropped)
            throw Exception("Table is dropped", ErrorCodes::TABLE_IS_DROPPED);
        return res;
    }


    /** Read a set of columns from the table.
      * Accepts a list of columns to read, as well as a description of the query,
      *  from which information can be extracted about how to retrieve data
      *  (indexes, locks, etc.)
      * Returns a stream with which you can read data sequentially
      *  or multiple streams for parallel data reading.
      * The into `processed_stage` info is also written to what stage the request was processed.
      * (Normally, the function only reads the columns from the list, but in other cases,
      *  for example, the request can be partially processed on a remote server.)
      *
      * settings - settings for one query.
      * Usually Storage does not care about these settings, since they are used in the interpreter.
      * But, for example, for distributed query processing, the settings are passed to the remote server.
      *
      * threads - a recommendation, how many threads to return,
      *  if the storage can return a different number of threads.
      *
      * It is guaranteed that the structure of the table will not change over the lifetime of the returned streams (that is, there will not be ALTER, RENAME and DROP).
      */
    virtual BlockInputStreams read(
        const Names & column_names,
        ASTPtr query,
        const Context & context,
        const Settings & settings,
        QueryProcessingStage::Enum & processed_stage,
        size_t max_block_size = DEFAULT_BLOCK_SIZE,
        unsigned threads = 1)
    {
        throw Exception("Method read is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Writes the data to a table.
      * Receives a description of the query, which can contain information about the data write method.
      * Returns an object by which you can write data sequentially.
      *
      * It is guaranteed that the table structure will not change over the lifetime of the returned streams (that is, there will not be ALTER, RENAME and DROP).
      */
    virtual BlockOutputStreamPtr write(
        ASTPtr query,
        const Settings & settings)
    {
        throw Exception("Method write is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Delete the table data. Called before deleting the directory with the data.
      * If you do not need any action other than deleting the directory with data, you can leave this method blank.
      */
    virtual void drop() {}

    /** Rename the table.
      * Renaming a name in a file with metadata, the name in the list of tables in the RAM, is done separately.
      * In this function, you need to rename the directory with the data, if any.
      * Called when the table structure is locked for write.
      */
    virtual void rename(const String & new_path_to_db, const String & new_database_name, const String & new_table_name)
    {
        throw Exception("Method rename is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** ALTER tables in the form of column changes that do not affect the change to Storage or its parameters.
      * This method must fully execute the ALTER query, taking care of the locks itself.
      * To update the table metadata on disk, this method should call InterpreterAlterQuery::updateMetadata.
      */
    virtual void alter(const AlterCommands & params, const String & database_name, const String & table_name, const Context & context)
    {
        throw Exception("Method alter is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Execute DROP COLUMN ... FROM PARTITION query witch drops column from given partition. */
    virtual void dropColumnFromPartition(ASTPtr query, const Field & partition, const Field & column_name, const Settings & settings)
    {
        throw Exception("Method dropColumnFromPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Run the query (DROP|DETACH) PARTITION.
      */
    virtual void dropPartition(ASTPtr query, const Field & partition, bool detach, bool unreplicated, const Settings & settings)
    {
        throw Exception("Method dropPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Run the ATTACH request [UNREPLICATED] (PART|PARTITION).
      */
    virtual void attachPartition(ASTPtr query, const Field & partition, bool unreplicated, bool part, const Settings & settings)
    {
        throw Exception("Method attachPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Run the FETCH PARTITION query.
      */
    virtual void fetchPartition(const Field & partition, const String & from, const Settings & settings)
    {
        throw Exception("Method fetchPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Run the FREEZE PARTITION request. That is, create a local backup (snapshot) of data using the `localBackup` function (see localBackup.h)
      */
    virtual void freezePartition(const Field & partition, const String & with_name, const Settings & settings)
    {
        throw Exception("Method freezePartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Run the RESHARD PARTITION query.
      */
    virtual void reshardPartitions(ASTPtr query, const String & database_name,
        const Field & first_partition, const Field & last_partition,
        const WeightedZooKeeperPaths & weighted_zookeeper_paths,
        const ASTPtr & sharding_key_expr, bool do_copy, const Field & coordinator,
        const Settings & settings)
    {
        throw Exception("Method reshardPartition is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** Perform any background work. For example, combining parts in a MergeTree type table.
      * Returns whether any work has been done.
      */
    virtual bool optimize(const String & partition, bool final, bool deduplicate, const Settings & settings)
    {
        throw Exception("Method optimize is not supported by storage " + getName(), ErrorCodes::NOT_IMPLEMENTED);
    }

    /** If you have to do some complicated work when destroying an object - do it in advance.
      * For example, if the table contains any threads for background work - ask them to complete and wait for completion.
      * By default, do nothing.
      * Can be called simultaneously from different threads, even after a call to drop().
      */
    virtual void shutdown() {}

    bool is_dropped{false};

    /// Does table support index for IN sections
    virtual bool supportsIndexForIn() const { return false; }

    /// Checks validity of the data
    virtual bool checkData() const { throw DB::Exception("Check query is not supported for " + getName() + " storage"); }

    /// Checks that table could be dropped right now
    /// If it can - returns true
    /// Otherwise - throws an exception with detailed information or returns false
    virtual bool checkTableCanBeDropped() const { return true; }

protected:
    using ITableDeclaration::ITableDeclaration;
    using std::enable_shared_from_this<IStorage>::shared_from_this;

private:
    friend class TableStructureReadLock;

    /// You always need to take the next two locks in this order.

    /** It is taken for read for the entire INSERT query and the entire merge of the parts (for MergeTree).
      * It is taken for write for the entire time ALTER MODIFY.
      *
      * Formally:
      * Taking a write lock ensures that:
      *  1) the data in the table will not change while the lock is alive,
      *  2) all changes to the data after releasing the lock will be based on the structure of the table at the time after the lock was released.
      * You need to take for read for the entire time of the operation that changes the data.
      */
    mutable Poco::RWLock data_lock;

    /** Lock for multiple columns and path to table. It is taken for write at RENAME, ALTER (for ALTER MODIFY for a while) and DROP.
      * It is taken for read for the whole time of SELECT, INSERT and merge parts (for MergeTree).
      *
      * Taking this lock for writing is a strictly "stronger" operation than taking parts_writing_lock for write record.
      * That is, if this lock is taken for write, you should not worry about `parts_writing_lock`.
      * parts_writing_lock is only needed for cases when you do not want to take `table_structure_lock` for long operations (ALTER MODIFY).
      */
    mutable Poco::RWLock structure_lock;
};

using StorageVector = std::vector<StoragePtr>;
using StorageList = std::list<StoragePtr>;

/// table name -> table
using Tables = std::map<String, StoragePtr>;

}