From fd05f7ea53ab59c0d5bd6fe1f44d337c5d321567 Mon Sep 17 00:00:00 2001 From: Michael Kolupaev Date: Wed, 15 Jan 2014 16:12:48 +0000 Subject: [PATCH] clickhouse: fixed a leak of file descriptors and memory. [#METR-9664] --- .../DB/Storages/MergeTree/MergeTreeReader.h | 11 ++++--- dbms/src/Storages/StorageLog.cpp | 30 +++++++++---------- dbms/src/Storages/StorageTinyLog.cpp | 14 +++++---- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h index 7e126e31975..d34086921e0 100644 --- a/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h +++ b/dbms/include/DB/Storages/MergeTree/MergeTreeReader.h @@ -221,8 +221,9 @@ private: String escaped_size_name = escapeForFileName(DataTypeNested::extractNestedTableName(name)) + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream( - path + escaped_size_name, uncompressed_cache))); + if (!streams.count(size_name)) + streams.insert(std::make_pair(size_name, new Stream( + path + escaped_size_name, uncompressed_cache))); addStream(name, *type_arr->getNestedType(), level + 1); } @@ -231,16 +232,14 @@ private: String size_name = name + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); String escaped_size_name = escaped_column_name + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream( - path + escaped_size_name, uncompressed_cache))); + streams[size_name] = new Stream(path + escaped_size_name, uncompressed_cache); const NamesAndTypesList & columns = *type_nested->getNestedTypesList(); for (NamesAndTypesList::const_iterator it = columns.begin(); it != columns.end(); ++it) addStream(DataTypeNested::concatenateNestedName(name, it->first), *it->second, level + 1); } else - streams.insert(std::make_pair(name, new Stream( - path + escaped_column_name, uncompressed_cache))); + streams[name] = new Stream(path + escaped_column_name, uncompressed_cache); } void readData(const String & name, const IDataType & type, IColumn & column, size_t from_mark, size_t max_rows_to_read, diff --git a/dbms/src/Storages/StorageLog.cpp b/dbms/src/Storages/StorageLog.cpp index ab216613bd8..1abc4f3a45e 100644 --- a/dbms/src/Storages/StorageLog.cpp +++ b/dbms/src/Storages/StorageLog.cpp @@ -109,33 +109,34 @@ void LogBlockInputStream::addStream(const String & name, const IDataType & type, if (const DataTypeArray * type_arr = dynamic_cast(&type)) { String size_name = DataTypeNested::extractNestedTableName(name) + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream( - storage.files[size_name].data_file.path(), - mark_number - ? storage.files[size_name].marks[mark_number].offset - : 0))); + if (!streams.count(size_name)) + streams.insert(std::make_pair(size_name, new Stream( + storage.files[size_name].data_file.path(), + mark_number + ? storage.files[size_name].marks[mark_number].offset + : 0))); addStream(name, *type_arr->getNestedType(), level + 1); } else if (const DataTypeNested * type_nested = dynamic_cast(&type)) { String size_name = name + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream( + streams[size_name] = new Stream( storage.files[size_name].data_file.path(), mark_number ? storage.files[size_name].marks[mark_number].offset - : 0))); + : 0); const NamesAndTypesList & columns = *type_nested->getNestedTypesList(); for (NamesAndTypesList::const_iterator it = columns.begin(); it != columns.end(); ++it) addStream(DataTypeNested::concatenateNestedName(name, it->first), *it->second, level + 1); } else - streams.insert(std::make_pair(name, new Stream( + streams[name] = new Stream( storage.files[name].data_file.path(), mark_number ? storage.files[name].marks[mark_number].offset - : 0))); + : 0); } @@ -234,24 +235,23 @@ void LogBlockOutputStream::addStream(const String & name, const IDataType & type if (const DataTypeArray * type_arr = dynamic_cast(&type)) { String size_name = DataTypeNested::extractNestedTableName(name) + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream( - storage.files[size_name].data_file.path()))); + if (!streams.count(size_name)) + streams.insert(std::make_pair(size_name, new Stream( + storage.files[size_name].data_file.path()))); addStream(name, *type_arr->getNestedType(), level + 1); } else if (const DataTypeNested * type_nested = dynamic_cast(&type)) { String size_name = name + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream( - storage.files[size_name].data_file.path()))); + streams[size_name] = new Stream(storage.files[size_name].data_file.path()); const NamesAndTypesList & columns = *type_nested->getNestedTypesList(); for (NamesAndTypesList::const_iterator it = columns.begin(); it != columns.end(); ++it) addStream(DataTypeNested::concatenateNestedName(name, it->first), *it->second, level + 1); } else - streams.insert(std::make_pair(name, new Stream( - storage.files[name].data_file.path()))); + streams[name] = new Stream(storage.files[name].data_file.path()); } diff --git a/dbms/src/Storages/StorageTinyLog.cpp b/dbms/src/Storages/StorageTinyLog.cpp index 22545f3aa78..425710ff847 100644 --- a/dbms/src/Storages/StorageTinyLog.cpp +++ b/dbms/src/Storages/StorageTinyLog.cpp @@ -102,21 +102,22 @@ void TinyLogBlockInputStream::addStream(const String & name, const IDataType & t if (const DataTypeArray * type_arr = dynamic_cast(&type)) { String size_name = DataTypeNested::extractNestedTableName(name) + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream(storage.files[size_name].data_file.path()))); + if (!streams.count(size_name)) + streams.insert(std::make_pair(size_name, new Stream(storage.files[size_name].data_file.path()))); addStream(name, *type_arr->getNestedType(), level + 1); } else if (const DataTypeNested * type_nested = dynamic_cast(&type)) { String size_name = name + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream(storage.files[size_name].data_file.path()))); + streams[size_name] = new Stream(storage.files[size_name].data_file.path()); const NamesAndTypesList & columns = *type_nested->getNestedTypesList(); for (NamesAndTypesList::const_iterator it = columns.begin(); it != columns.end(); ++it) addStream(DataTypeNested::concatenateNestedName(name, it->first), *it->second, level + 1); } else - streams.insert(std::make_pair(name, new Stream(storage.files[name].data_file.path()))); + streams[name] = new Stream(storage.files[name].data_file.path()); } @@ -185,21 +186,22 @@ void TinyLogBlockOutputStream::addStream(const String & name, const IDataType & if (const DataTypeArray * type_arr = dynamic_cast(&type)) { String size_name = DataTypeNested::extractNestedTableName(name) + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream(storage.files[size_name].data_file.path()))); + if (streams.count(size_name)) + streams.insert(std::make_pair(size_name, new Stream(storage.files[size_name].data_file.path()))); addStream(name, *type_arr->getNestedType(), level + 1); } else if (const DataTypeNested * type_nested = dynamic_cast(&type)) { String size_name = name + ARRAY_SIZES_COLUMN_NAME_SUFFIX + toString(level); - streams.insert(std::make_pair(size_name, new Stream(storage.files[size_name].data_file.path()))); + streams[size_name] = new Stream(storage.files[size_name].data_file.path()); const NamesAndTypesList & columns = *type_nested->getNestedTypesList(); for (NamesAndTypesList::const_iterator it = columns.begin(); it != columns.end(); ++it) addStream(DataTypeNested::concatenateNestedName(name, it->first), *it->second, level + 1); } else - streams.insert(std::make_pair(name, new Stream(storage.files[name].data_file.path()))); + streams[name] = new Stream(storage.files[name].data_file.path()); }