From ec156e9f2be1bb55aa6392eb1c5cb47c96d9eb16 Mon Sep 17 00:00:00 2001 From: alesapin Date: Fri, 20 Jul 2018 13:00:56 +0300 Subject: [PATCH] CLICKHOUSE-3772: Add system tables formats, data_type_families, table_functions, aggregate_function_combinators --- .../AggregateFunctionCombinatorFactory.h | 4 + dbms/src/DataTypes/DataTypeFactory.h | 4 + dbms/src/Formats/FormatFactory.h | 4 + .../System/IStorageSystemWithStringColumns.h | 53 +++++++++++ ...rageSystemAggregateFunctionCombinators.cpp | 14 +++ ...torageSystemAggregateFunctionCombinators.h | 26 ++++++ .../System/StorageSystemDataTypeFamilies.cpp | 91 +++++++++++++++++++ .../System/StorageSystemDataTypeFamilies.h | 25 +++++ .../Storages/System/StorageSystemFormats.cpp | 30 ++++++ .../Storages/System/StorageSystemFormats.h | 26 ++++++ .../System/StorageSystemTableFunctions.cpp | 14 +++ .../System/StorageSystemTableFunctions.h | 26 ++++++ .../Storages/System/attachSystemTables.cpp | 8 ++ .../src/TableFunctions/TableFunctionFactory.h | 7 +- 14 files changed, 330 insertions(+), 2 deletions(-) create mode 100644 dbms/src/Storages/System/IStorageSystemWithStringColumns.h create mode 100644 dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.cpp create mode 100644 dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.h create mode 100644 dbms/src/Storages/System/StorageSystemDataTypeFamilies.cpp create mode 100644 dbms/src/Storages/System/StorageSystemDataTypeFamilies.h create mode 100644 dbms/src/Storages/System/StorageSystemFormats.cpp create mode 100644 dbms/src/Storages/System/StorageSystemFormats.h create mode 100644 dbms/src/Storages/System/StorageSystemTableFunctions.cpp create mode 100644 dbms/src/Storages/System/StorageSystemTableFunctions.h diff --git a/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h b/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h index 4c70cc6c068..dc3e21cdc53 100644 --- a/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h +++ b/dbms/src/AggregateFunctions/AggregateFunctionCombinatorFactory.h @@ -22,6 +22,10 @@ public: /// Example: if the name is 'avgIf', it will return combinator -If. AggregateFunctionCombinatorPtr tryFindSuffix(const std::string & name) const; + std::unordered_map getAllAggregateFunctionCombinators() const { + return dict; + } + private: std::unordered_map dict; }; diff --git a/dbms/src/DataTypes/DataTypeFactory.h b/dbms/src/DataTypes/DataTypeFactory.h index cad176432de..c35a72faf24 100644 --- a/dbms/src/DataTypes/DataTypeFactory.h +++ b/dbms/src/DataTypes/DataTypeFactory.h @@ -44,6 +44,10 @@ public: /// Register a simple data type, that have no parameters. void registerSimpleDataType(const String & name, SimpleCreator creator, CaseSensitiveness case_sensitiveness = CaseSensitive); + DataTypesDictionary getAllDataTypes() const { + return data_types; + } + private: DataTypesDictionary data_types; diff --git a/dbms/src/Formats/FormatFactory.h b/dbms/src/Formats/FormatFactory.h index f415e97abdb..8a446cc393a 100644 --- a/dbms/src/Formats/FormatFactory.h +++ b/dbms/src/Formats/FormatFactory.h @@ -58,6 +58,10 @@ public: void registerInputFormat(const String & name, InputCreator input_creator); void registerOutputFormat(const String & name, OutputCreator output_creator); + const FormatsDictionary & getAllFormats() const { + return dict; + } + private: FormatsDictionary dict; diff --git a/dbms/src/Storages/System/IStorageSystemWithStringColumns.h b/dbms/src/Storages/System/IStorageSystemWithStringColumns.h new file mode 100644 index 00000000000..ff12cf3cd98 --- /dev/null +++ b/dbms/src/Storages/System/IStorageSystemWithStringColumns.h @@ -0,0 +1,53 @@ +#pragma once +#include +#include +#include +#include +#include + +namespace DB +{ +class Context; +template +class IStorageSystemWithStringColumns : public IStorage +{ +protected: + virtual void fillData(MutableColumns & res_columns) const = 0; + +public: + IStorageSystemWithStringColumns (const String & name_) : name(name_) + { + auto names = Self::getColumnNames(); + NamesAndTypesList name_list; + for (const auto & name : names) + { + name_list.push_back(NameAndTypePair{name, std::make_shared()}); + } + setColumns(ColumnsDescription(name_list)); + } + + std::string getTableName() const override + { + return name; + } + + BlockInputStreams read(const Names & column_names, + const SelectQueryInfo & /*query_info*/, + const Context & /*context*/, + QueryProcessingStage::Enum & processed_stage, + size_t /*max_block_size*/, + unsigned /*num_streams*/) override + { + check(column_names); + processed_stage = QueryProcessingStage::FetchColumns; + + MutableColumns res_columns = getSampleBlock().cloneEmptyColumns(); + fillData(res_columns); + + return BlockInputStreams(1, std::make_shared(getSampleBlock().cloneWithColumns(std::move(res_columns)))); + } + +private: + const String name; +}; +} diff --git a/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.cpp b/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.cpp new file mode 100644 index 00000000000..9dd106ce2d7 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.cpp @@ -0,0 +1,14 @@ +#include +#include + +namespace DB +{ +void StorageSystemAggregateFunctionCombinators::fillData(MutableColumns & res_columns) const +{ + const auto & combinators = AggregateFunctionCombinatorFactory::instance().getAllAggregateFunctionCombinators(); + for (const auto & pair : combinators) + { + res_columns[0]->insert(pair.first); + } +} +} diff --git a/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.h b/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.h new file mode 100644 index 00000000000..097fe93666e --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemAggregateFunctionCombinators.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +namespace DB +{ +class StorageSystemAggregateFunctionCombinators : public ext::shared_ptr_helper, + public IStorageSystemWithStringColumns +{ +protected: + void fillData(MutableColumns & res_columns) const override; + +public: + using IStorageSystemWithStringColumns::IStorageSystemWithStringColumns; + + std::string getName() const override + { + return "SystemAggregateFunctionCombinators"; + } + + static std::vector getColumnNames() + { + return {"name"}; + } +}; +} diff --git a/dbms/src/Storages/System/StorageSystemDataTypeFamilies.cpp b/dbms/src/Storages/System/StorageSystemDataTypeFamilies.cpp new file mode 100644 index 00000000000..3fdb8ebb8f5 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemDataTypeFamilies.cpp @@ -0,0 +1,91 @@ +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB +{ +namespace +{ + String getPropertiesAsString(const DataTypePtr data_type) + { + std::vector properties; + if (data_type->isParametric()) + properties.push_back("parametric"); + if (data_type->haveSubtypes()) + properties.push_back("have_subtypes"); + if (data_type->cannotBeStoredInTables()) + properties.push_back("cannot_be_stored_in_tables"); + if (data_type->isComparable()) + properties.push_back("comparable"); + if (data_type->canBeComparedWithCollation()) + properties.push_back("can_be_compared_with_collation"); + if (data_type->canBeUsedAsVersion()) + properties.push_back("can_be_used_as_version"); + if (data_type->isSummable()) + properties.push_back("summable"); + if (data_type->canBeUsedInBitOperations()) + properties.push_back("can_be_used_in_bit_operations"); + if (data_type->canBeUsedInBooleanContext()) + properties.push_back("can_be_used_in_boolean_context"); + if (data_type->isValueRepresentedByNumber()) + properties.push_back("value_represented_by_number"); + if (data_type->isCategorial()) + properties.push_back("categorial"); + if (data_type->isNullable()) + properties.push_back("nullable"); + if (data_type->onlyNull()) + properties.push_back("only_null"); + if (data_type->canBeInsideNullable()) + properties.push_back("can_be_inside_nullable"); + return boost::algorithm::join(properties, ","); + } + ASTPtr createFakeEnumCreationAst() + { + String fakename = "e"; + ASTPtr name = std::make_shared(Field(fakename.c_str(), fakename.size())); + ASTPtr value = std::make_shared(Field(UInt64(1))); + ASTPtr ast_func = makeASTFunction("equals", name, value); + ASTPtr clone = ast_func->clone(); + clone->children.clear(); + clone->children.push_back(ast_func); + return clone; + } +} + +void StorageSystemDataTypeFamilies::fillData(MutableColumns & res_columns) const +{ + const auto & factory = DataTypeFactory::instance(); + const auto & data_types = factory.getAllDataTypes(); + for (const auto & pair : data_types) + { + res_columns[0]->insert(pair.first); + + try + { + DataTypePtr type_ptr; + //special case with enum, because it has arguments but it's properties doesn't + //depend on arguments + if (boost::starts_with(pair.first, "Enum")) + { + type_ptr = factory.get(pair.first, createFakeEnumCreationAst()); + } + else + { + type_ptr = factory.get(pair.first); + } + + res_columns[1]->insert(getPropertiesAsString(type_ptr)); + } + catch (Exception & ex) + { + res_columns[1]->insert(String{"depends_on_arguments"}); + } + } +} +} diff --git a/dbms/src/Storages/System/StorageSystemDataTypeFamilies.h b/dbms/src/Storages/System/StorageSystemDataTypeFamilies.h new file mode 100644 index 00000000000..38b769e6e1c --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemDataTypeFamilies.h @@ -0,0 +1,25 @@ +#pragma once +#include +#include +namespace DB +{ +class StorageSystemDataTypeFamilies : public ext::shared_ptr_helper, + public IStorageSystemWithStringColumns +{ +protected: + void fillData(MutableColumns & res_columns) const override; + +public: + using IStorageSystemWithStringColumns::IStorageSystemWithStringColumns; + + std::string getName() const override + { + return "SystemTableDataTypeFamilies"; + } + + static std::vector getColumnNames() + { + return {"name", "properties"}; + } +}; +} diff --git a/dbms/src/Storages/System/StorageSystemFormats.cpp b/dbms/src/Storages/System/StorageSystemFormats.cpp new file mode 100644 index 00000000000..b029e354cc2 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemFormats.cpp @@ -0,0 +1,30 @@ +#include +#include + +namespace DB +{ +void StorageSystemFormats::fillData(MutableColumns & res_columns) const +{ + const auto & formats = FormatFactory::instance().getAllFormats(); + for (const auto & pair : formats) + { + const auto & [name, creator_pair] = pair; + bool has_input_format = (creator_pair.first != nullptr); + bool has_output_format = (creator_pair.second != nullptr); + res_columns[0]->insert(name); + std::string format_type; + if (has_input_format) + format_type = "input"; + + if (has_output_format) + { + if (!format_type.empty()) + format_type += "/output"; + else + format_type = "output"; + } + + res_columns[1]->insert(format_type); + } +} +} diff --git a/dbms/src/Storages/System/StorageSystemFormats.h b/dbms/src/Storages/System/StorageSystemFormats.h new file mode 100644 index 00000000000..90f6cb36cc9 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemFormats.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include + +namespace DB +{ +class StorageSystemFormats : public ext::shared_ptr_helper, public IStorageSystemWithStringColumns +{ +protected: + void fillData(MutableColumns & res_columns) const override; +public: + using IStorageSystemWithStringColumns::IStorageSystemWithStringColumns; + + std::string getName() const override + { + return "SystemFormats"; + } + + static std::vector getColumnNames() + { + return {"name", "description"}; + } + +}; +} diff --git a/dbms/src/Storages/System/StorageSystemTableFunctions.cpp b/dbms/src/Storages/System/StorageSystemTableFunctions.cpp new file mode 100644 index 00000000000..aaf448a6559 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemTableFunctions.cpp @@ -0,0 +1,14 @@ +#include + +#include +namespace DB +{ +void StorageSystemTableFunctions::fillData(MutableColumns & res_columns) const +{ + const auto & functions = TableFunctionFactory::instance().getAllTableFunctions(); + for (const auto & pair : functions) + { + res_columns[0]->insert(pair.first); + } +} +} diff --git a/dbms/src/Storages/System/StorageSystemTableFunctions.h b/dbms/src/Storages/System/StorageSystemTableFunctions.h new file mode 100644 index 00000000000..096b3d8f4a4 --- /dev/null +++ b/dbms/src/Storages/System/StorageSystemTableFunctions.h @@ -0,0 +1,26 @@ +#pragma once + +#include +#include +namespace DB +{ +class StorageSystemTableFunctions : public ext::shared_ptr_helper, + public IStorageSystemWithStringColumns +{ +protected: + void fillData(MutableColumns & res_columns) const override; + +public: + using IStorageSystemWithStringColumns::IStorageSystemWithStringColumns; + + std::string getName() const override + { + return "SystemTableFunctions"; + } + + static std::vector getColumnNames() + { + return {"name"}; + } +}; +} diff --git a/dbms/src/Storages/System/attachSystemTables.cpp b/dbms/src/Storages/System/attachSystemTables.cpp index 705d01fb9c2..ef7ed8d7da5 100644 --- a/dbms/src/Storages/System/attachSystemTables.cpp +++ b/dbms/src/Storages/System/attachSystemTables.cpp @@ -1,13 +1,16 @@ #include #include +#include #include #include #include #include #include +#include #include #include +#include #include #include #include @@ -23,6 +26,7 @@ #include #include #include +#include #include #include @@ -42,6 +46,10 @@ void attachSystemTablesLocal(IDatabase & system_database) system_database.attachTable("events", StorageSystemEvents::create("events")); system_database.attachTable("settings", StorageSystemSettings::create("settings")); system_database.attachTable("build_options", StorageSystemBuildOptions::create("build_options")); + system_database.attachTable("formats", StorageSystemFormats::create("formats")); + system_database.attachTable("table_functions", StorageSystemTableFunctions::create("table_functions")); + system_database.attachTable("aggregate_function_combinators", StorageSystemAggregateFunctionCombinators::create("aggregate_function_combinators")); + system_database.attachTable("data_type_families", StorageSystemDataTypeFamilies::create("data_type_families")); } void attachSystemTablesServer(IDatabase & system_database, bool has_zookeeper) diff --git a/dbms/src/TableFunctions/TableFunctionFactory.h b/dbms/src/TableFunctions/TableFunctionFactory.h index 27c0f566a78..65361a59083 100644 --- a/dbms/src/TableFunctions/TableFunctionFactory.h +++ b/dbms/src/TableFunctions/TableFunctionFactory.h @@ -23,6 +23,7 @@ class TableFunctionFactory final: public ext::singleton public: using Creator = std::function; + using TableFunctions = std::unordered_map; /// Register a function by its name. /// No locking, you must register all functions before usage of get. void registerFunction(const std::string & name, Creator creator); @@ -42,9 +43,11 @@ public: const std::string & name, const Context & context) const; -private: - using TableFunctions = std::unordered_map; + TableFunctions getAllTableFunctions() const { + return functions; + } +private: TableFunctions functions; };