From ce391c84c5460e997262970372f08ab52e8ea01d Mon Sep 17 00:00:00 2001 From: Bharat Nallan Date: Tue, 23 Jun 2020 18:11:06 -0700 Subject: [PATCH 1/6] remove unused imports --- src/Databases/DatabaseOrdinary.cpp | 5 ----- 1 file changed, 5 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index eec58ed9b33..74b4bd87eb1 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -1,5 +1,3 @@ -#include - #include #include #include @@ -12,17 +10,14 @@ #include #include #include -#include #include #include #include #include -#include #include #include -#include #include #include #include From 6a9f3d768a8aede9776d218ccdabf3ce22eca9f2 Mon Sep 17 00:00:00 2001 From: Bharat Nallan Date: Tue, 23 Jun 2020 18:31:25 -0700 Subject: [PATCH 2/6] clang format file --- src/Databases/DatabaseOrdinary.cpp | 69 ++++++++++++++---------------- 1 file changed, 33 insertions(+), 36 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 74b4bd87eb1..fd689005a36 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -9,26 +10,22 @@ #include #include #include -#include -#include -#include #include -#include - +#include +#include +#include #include - #include #include -#include #include #include +#include #include #include namespace DB { - static constexpr size_t PRINT_MESSAGE_EACH_N_OBJECTS = 256; static constexpr size_t PRINT_MESSAGE_EACH_N_SECONDS = 5; static constexpr size_t METADATA_FILE_BUFFER_SIZE = 32768; @@ -54,18 +51,15 @@ namespace } catch (Exception & e) { - e.addMessage("Cannot attach table " + backQuote(database_name) + "." + backQuote(query.table) - + " from metadata file " + metadata_path + e.addMessage( + "Cannot attach table " + backQuote(database_name) + "." + backQuote(query.table) + " from metadata file " + metadata_path + " from query " + serializeAST(query)); throw; } } - void tryAttachDictionary( - const ASTPtr & query, - DatabaseOrdinary & database, - const String & metadata_path) + void tryAttachDictionary(const ASTPtr & query, DatabaseOrdinary & database, const String & metadata_path) { auto & create_query = query->as(); assert(create_query.is_dictionary); @@ -78,9 +72,9 @@ namespace } catch (Exception & e) { - e.addMessage("Cannot attach dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(create_query.table) + - " from metadata file " + metadata_path + - " from query " + serializeAST(*query)); + e.addMessage( + "Cannot attach dictionary " + backQuote(database.getDatabaseName()) + "." + backQuote(create_query.table) + + " from metadata file " + metadata_path + " from query " + serializeAST(*query)); throw; } } @@ -102,14 +96,13 @@ DatabaseOrdinary::DatabaseOrdinary(const String & name_, const String & metadata { } -DatabaseOrdinary::DatabaseOrdinary(const String & name_, const String & metadata_path_, const String & data_path_, const String & logger, const Context & context_) - : DatabaseWithDictionaries(name_, metadata_path_, data_path_, logger, context_) +DatabaseOrdinary::DatabaseOrdinary( + const String & name_, const String & metadata_path_, const String & data_path_, const String & logger, const Context & context_) + : DatabaseWithDictionaries(name_, metadata_path_, data_path_, logger, context_) { } -void DatabaseOrdinary::loadStoredObjects( - Context & context, - bool has_force_restore_data_flag) +void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_restore_data_flag) { /** Tables load faster if they are loaded in sorted (by name) order. * Otherwise (for the ext4 filesystem), `DirectoryIterator` iterates through them in some order, @@ -120,8 +113,7 @@ void DatabaseOrdinary::loadStoredObjects( size_t total_dictionaries = 0; - auto process_metadata = [&context, &file_names, &total_dictionaries, this](const String & file_name) - { + auto process_metadata = [&context, &file_names, &total_dictionaries, this](const String & file_name) { String full_path = getMetadataPath() + file_name; try { @@ -138,7 +130,6 @@ void DatabaseOrdinary::loadStoredObjects( e.addMessage("Cannot parse definition from metadata file " + full_path); throw; } - }; iterateMetadataFiles(context, process_metadata); @@ -158,9 +149,14 @@ void DatabaseOrdinary::loadStoredObjects( { const auto & create_query = name_with_query.second->as(); if (!create_query.is_dictionary) - pool.scheduleOrThrowOnError([&]() - { - tryAttachTable(context, create_query, *this, getDatabaseName(), getMetadataPath() + name_with_query.first, has_force_restore_data_flag); + pool.scheduleOrThrowOnError([&]() { + tryAttachTable( + context, + create_query, + *this, + getDatabaseName(), + getMetadataPath() + name_with_query.first, + has_force_restore_data_flag); /// Messages, so that it's not boring to wait for the server to load for a long time. logAboutProgress(log, ++tables_processed, total_tables, watch); @@ -198,8 +194,7 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool) AtomicStopwatch watch; std::atomic tables_processed{0}; - auto startup_one_table = [&](const StoragePtr & table) - { + auto startup_one_table = [&](const StoragePtr & table) { table->startup(); logAboutProgress(log, ++tables_processed, total_tables, watch); }; @@ -217,10 +212,7 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool) thread_pool.wait(); } -void DatabaseOrdinary::alterTable( - const Context & context, - const StorageID & table_id, - const StorageInMemoryMetadata & metadata) +void DatabaseOrdinary::alterTable(const Context & context, const StorageID & table_id, const StorageInMemoryMetadata & metadata) { String table_name = table_id.table_name; /// Read the definition of the table and replace the necessary parts with new ones. @@ -235,8 +227,13 @@ void DatabaseOrdinary::alterTable( } ParserCreateQuery parser; - ASTPtr ast = parseQuery(parser, statement.data(), statement.data() + statement.size(), "in file " + table_metadata_path, - 0, context.getSettingsRef().max_parser_depth); + ASTPtr ast = parseQuery( + parser, + statement.data(), + statement.data() + statement.size(), + "in file " + table_metadata_path, + 0, + context.getSettingsRef().max_parser_depth); auto & ast_create_query = ast->as(); From ea40e7b4039f69e8e1cb20ce23039ed0c93bc7ca Mon Sep 17 00:00:00 2001 From: Bharat Nallan Date: Tue, 23 Jun 2020 18:43:36 -0700 Subject: [PATCH 3/6] construct path using boost::filesystem::path --- src/Databases/DatabaseOrdinary.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index fd689005a36..848029f7dd2 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -1,3 +1,5 @@ +#include + #include #include #include @@ -23,6 +25,7 @@ #include #include +namespace fs = boost::filesystem; namespace DB { @@ -114,10 +117,13 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto size_t total_dictionaries = 0; auto process_metadata = [&context, &file_names, &total_dictionaries, this](const String & file_name) { - String full_path = getMetadataPath() + file_name; + fs::path path(getMetadataPath()); + fs::path file_path(file_name); + fs::path full_path = path / file_path; + try { - auto ast = parseQueryFromMetadata(log, context, full_path, /*throw_on_error*/ true, /*remove_empty*/ false); + auto ast = parseQueryFromMetadata(log, context, full_path.string(), /*throw_on_error*/ true, /*remove_empty*/ false); if (ast) { auto * create_query = ast->as(); @@ -127,7 +133,7 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto } catch (Exception & e) { - e.addMessage("Cannot parse definition from metadata file " + full_path); + e.addMessage("Cannot parse definition from metadata file " + full_path.string()); throw; } }; From 1244ca562759c4b92a08f4350080cc01567e8c99 Mon Sep 17 00:00:00 2001 From: Bharat Nallan Date: Wed, 24 Jun 2020 07:34:32 -0700 Subject: [PATCH 4/6] fix style check - clang format for lambdas --- src/Databases/DatabaseOrdinary.cpp | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 848029f7dd2..76023f92c3c 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -116,7 +116,9 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto size_t total_dictionaries = 0; - auto process_metadata = [&context, &file_names, &total_dictionaries, this](const String & file_name) { + // clang-format off + auto process_metadata = [&context, &file_names, &total_dictionaries, this](const String & file_name) + { fs::path path(getMetadataPath()); fs::path file_path(file_name); fs::path full_path = path / file_path; @@ -137,6 +139,8 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto throw; } }; + // clang-format on + iterateMetadataFiles(context, process_metadata); @@ -155,7 +159,9 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto { const auto & create_query = name_with_query.second->as(); if (!create_query.is_dictionary) - pool.scheduleOrThrowOnError([&]() { + // clang-format off + pool.scheduleOrThrowOnError([&]() + { tryAttachTable( context, create_query, @@ -167,6 +173,7 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto /// Messages, so that it's not boring to wait for the server to load for a long time. logAboutProgress(log, ++tables_processed, total_tables, watch); }); + // clang-format on } pool.wait(); @@ -200,10 +207,14 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool) AtomicStopwatch watch; std::atomic tables_processed{0}; - auto startup_one_table = [&](const StoragePtr & table) { + // clang-format off + auto startup_one_table = [&](const StoragePtr & table) + { table->startup(); logAboutProgress(log, ++tables_processed, total_tables, watch); }; + // clang-format on + try { From 325a515e90da04e099e14ccdaa21ee91f7c76d5d Mon Sep 17 00:00:00 2001 From: Bharat Nallan Date: Wed, 24 Jun 2020 12:01:12 -0700 Subject: [PATCH 5/6] remove clang-format tag comments --- src/Databases/DatabaseOrdinary.cpp | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index 76023f92c3c..c610d2d2eee 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -116,7 +116,6 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto size_t total_dictionaries = 0; - // clang-format off auto process_metadata = [&context, &file_names, &total_dictionaries, this](const String & file_name) { fs::path path(getMetadataPath()); @@ -139,7 +138,6 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto throw; } }; - // clang-format on iterateMetadataFiles(context, process_metadata); @@ -159,7 +157,6 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto { const auto & create_query = name_with_query.second->as(); if (!create_query.is_dictionary) - // clang-format off pool.scheduleOrThrowOnError([&]() { tryAttachTable( @@ -173,7 +170,6 @@ void DatabaseOrdinary::loadStoredObjects(Context & context, bool has_force_resto /// Messages, so that it's not boring to wait for the server to load for a long time. logAboutProgress(log, ++tables_processed, total_tables, watch); }); - // clang-format on } pool.wait(); @@ -207,13 +203,11 @@ void DatabaseOrdinary::startupTables(ThreadPool & thread_pool) AtomicStopwatch watch; std::atomic tables_processed{0}; - // clang-format off auto startup_one_table = [&](const StoragePtr & table) { table->startup(); logAboutProgress(log, ++tables_processed, total_tables, watch); }; - // clang-format on try From 76938b40a46eeb1dd344a74d0d26ed96ac968a01 Mon Sep 17 00:00:00 2001 From: bharatnc Date: Thu, 25 Jun 2020 18:30:44 -0700 Subject: [PATCH 6/6] use std::filesystem::path --- src/Databases/DatabaseOrdinary.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp index c610d2d2eee..9194558dffb 100644 --- a/src/Databases/DatabaseOrdinary.cpp +++ b/src/Databases/DatabaseOrdinary.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include @@ -25,7 +25,7 @@ #include #include -namespace fs = boost::filesystem; +namespace fs = std::filesystem; namespace DB {