thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-26 01:22:04 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fix lock-order-inversion between periodic dictionary reload and config reload

Browse Source 
Integration tests found [1], TSan report:

    WARNING: ThreadSanitizer: lock-order-inversion (potential deadlock) (pid=11)
      Cycle in lock order graph: M3152 (0x7b9000000058) => M3153 (0x7b9000000438) => M3152

      Mutex M3153 acquired here while holding mutex M3152 in main thread:
        3 std::__1::lock_guard<std::__1::recursive_mutex>::lock_guard(std::__1::recursive_mutex&) obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:91:27 (clickhouse+0x15716b81)
        4 DB::ExternalLoader::addConfigRepository() const obj-x86_64-linux-gnu/../src/Interpreters/ExternalLoader.cpp:1283:21 (clickhouse+0x15716b81)
        5 DB::Context::loadOrReloadDictionaries() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1453:62 (clickhouse+0x155d7bad)
        6 DB::Server::main()::$_1::operator()() const obj-x86_64-linux-gnu/../programs/server/Server.cpp:852:29 (clickhouse+0x9b1944c)
        13 DB::ConfigReloader::reloadIfNewer() obj-x86_64-linux-gnu/../src/Common/Config/ConfigReloader.cpp:137:13 (clickhouse+0x17045e2e)
        14 DB::ConfigReloader::ConfigReloader() obj-x86_64-linux-gnu/../src/Common/Config/ConfigReloader.cpp:33:9 (clickhouse+0x17044e51)
        16 DB::Server::main(std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > > const&) obj-x86_64-linux-gnu/../programs/server/Server.cpp:803:33 (clickhouse+0x9b0c41d)
        17 Poco::Util::Application::run() obj-x86_64-linux-gnu/../contrib/poco/Util/src/Application.cpp:334:8 (clickhouse+0x19ffc08b)
        18 DB::Server::run() obj-x86_64-linux-gnu/../programs/server/Server.cpp:405:25 (clickhouse+0x9b03ebe)
        19 Poco::Util::ServerApplication::run(int, char**) obj-x86_64-linux-gnu/../contrib/poco/Util/src/ServerApplication.cpp:611:9 (clickhouse+0x1a01c246)
        20 mainEntryClickHouseServer(int, char**) obj-x86_64-linux-gnu/../programs/server/Server.cpp:183:20 (clickhouse+0x9b02576)
        21 main obj-x86_64-linux-gnu/../programs/main.cpp:372:12 (clickhouse+0x9b00a3a)

      Mutex M3152 acquired here while holding mutex M3153 in thread T2:
        3 std::__1::lock_guard<std::__1::mutex>::lock_guard() obj-x86_64-linux-gnu/../contrib/libcxx/include/__mutex_base:91:27 (clickhouse+0x155d63b8)
        4 DB::Context::getExternalDictionariesLoader() obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1337:21 (clickhouse+0x155d63b8)
        5 DB::Context::getExternalDictionariesLoader() const obj-x86_64-linux-gnu/../src/Interpreters/Context.cpp:1332:41 (clickhouse+0x155d6359)
        6 DB::DatabaseDictionary::tryGetTable() const obj-x86_64-inux-gnu/../src/Databases/DatabaseDictionary.cpp:76:38 (clickhouse+0x157819ad)
        7 DB::DatabaseCatalog::getTableImpl() const obj-x86_64-linux-gnu/../src/Interpreters/DatabaseCatalog.cpp:285:28 (clickhouse+0x1564a1fa)
        8 DB::DatabaseCatalog::getTable() const obj-x86_64-linux-gnu/../src/Interpreters/DatabaseCatalog.cpp:656:16 (clickhouse+0x1564fa2a)
        9 DB::JoinedTables::getLeftTableStorage() obj-x86_64-linux-gnu/../src/Interpreters/JoinedTables.cpp:219:40 (clickhouse+0x15eeef45)
        10 DB::InterpreterSelectQuery::InterpreterSelectQuery() obj-x86_64-linux-gnu/../src/Interpreters/InterpreterSelectQuery.cpp:321:33 (clickhouse+0x15b792be)
        19 DB::ClickHouseDictionarySource::doInvalidateQuery() const obj-x86_64-linux-gnu/../src/Dictionaries/ClickHouseDictionarySource.cpp:207:36 (clickhouse+0x12872d2d)
        20 DB::ClickHouseDictionarySource::isModified() const obj-x86_64-linux-gnu/../src/Dictionaries/ClickHouseDictionarySource.cpp:144:25 (clickhouse+0x12872534)
        21 DB::IDictionary::isModified() const <null> (clickhouse+0x128ce39b)
        22 DB::ExternalLoader::LoadingDispatcher::reloadOutdated() obj-x86_64-linux-gnu/../src/Interpreters/ExternalLoader.cpp:660:50 (clickhouse+0x157305f7)
        23 DB::ExternalLoader::PeriodicUpdater::doPeriodicUpdates() obj-x86_64-linux-gnu/../src/Interpreters/ExternalLoader.cpp:1248:36 (clickhouse+0x1572fff7)

  [1]: https://clickhouse-test-reports.s3.yandex.net/29856/42ca2b4bb241827edf69bbd6938d6b19c31935f1/integration_tests_(thread).html#fail1
This commit is contained in:
Azat Khuzhin 2021-10-10 00:46:28 +03:00
parent 2c3b866ab1
commit 2bab572caf
1 changed files with 4 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/Interpreters/ExternalLoader.cpp
View File

@ -1243,8 +1243,10 @@ private:
{
lock.unlock();
{
std::lock_guard config_lock{config_mutex};
loading_dispatcher.setConfiguration(config_files_reader.read());
{
std::lock_guard config_lock{config_mutex};
loading_dispatcher.setConfiguration(config_files_reader.read());
}
loading_dispatcher.reloadOutdated();
}
lock.lock();