diff --git a/.clang-tidy b/.clang-tidy
index de19059d09e..bb63bf2eea6 100644
--- a/.clang-tidy
+++ b/.clang-tidy
@@ -27,6 +27,8 @@ Checks: [
'-bugprone-not-null-terminated-result',
'-bugprone-reserved-identifier', # useful but too slow, TODO retry when https://reviews.llvm.org/rG1c282052624f9d0bd273bde0b47b30c96699c6c7 is merged
'-bugprone-unchecked-optional-access',
+ '-bugprone-crtp-constructor-accessibility',
+ '-bugprone-suspicious-stringview-data-usage',
'-cert-dcl16-c',
'-cert-dcl37-c',
@@ -36,6 +38,7 @@ Checks: [
'-cert-msc51-cpp',
'-cert-oop54-cpp',
'-cert-oop57-cpp',
+ '-cert-err33-c', # Misreports on clang-19: it warns about all functions containing 'remove' in the name, not only about the standard library.
'-clang-analyzer-optin.performance.Padding',
@@ -99,6 +102,7 @@ Checks: [
'-modernize-use-emplace',
'-modernize-use-nodiscard',
'-modernize-use-trailing-return-type',
+ '-modernize-use-designated-initializers',
'-performance-enum-size',
'-performance-inefficient-string-concatenation',
diff --git a/contrib/grpc b/contrib/grpc
index f5b7fdc2dff..1716359d2e2 160000
--- a/contrib/grpc
+++ b/contrib/grpc
@@ -1 +1 @@
-Subproject commit f5b7fdc2dff09ada06dbf6c75df298fb40f898df
+Subproject commit 1716359d2e28d304a250f9df0e6c0ccad03de8db
diff --git a/docs/en/operations/system-tables/detached_tables.md b/docs/en/operations/system-tables/detached_tables.md
new file mode 100644
index 00000000000..f669c8fd251
--- /dev/null
+++ b/docs/en/operations/system-tables/detached_tables.md
@@ -0,0 +1,35 @@
+---
+slug: /en/operations/system-tables/detached_tables
+---
+# detached_tables
+
+Contains information of each detached table.
+
+Columns:
+
+- `database` ([String](../../sql-reference/data-types/string.md)) — The name of the database the table is in.
+
+- `table` ([String](../../sql-reference/data-types/string.md)) — Table name.
+
+- `uuid` ([UUID](../../sql-reference/data-types/uuid.md)) — Table uuid (Atomic database).
+
+- `metadata_path` ([String](../../sql-reference/data-types/string.md)) - Path to the table metadata in the file system.
+
+- `is_permanently` ([UInt8](../../sql-reference/data-types/int-uint.md)) - Flag indicates that the table was detached PERMANENTLY.
+
+
+**Example**
+
+```sql
+SELECT * FROM system.detached_tables FORMAT Vertical;
+```
+
+```text
+Row 1:
+──────
+database: base
+table: t1
+uuid: 81b1c20a-b7c6-4116-a2ce-7583fb6b6736
+metadata_path: /var/lib/clickhouse/store/461/461cf698-fd0b-406d-8c01-5d8fd5748a91/t1.sql
+is_permanently: 1
+```
diff --git a/docs/en/sql-reference/data-types/lowcardinality.md b/docs/en/sql-reference/data-types/lowcardinality.md
index 133ac2bd72e..a92f6f4dce6 100644
--- a/docs/en/sql-reference/data-types/lowcardinality.md
+++ b/docs/en/sql-reference/data-types/lowcardinality.md
@@ -56,7 +56,6 @@ Functions:
## Related content
-- [Reducing ClickHouse Storage Cost with the Low Cardinality Type – Lessons from an Instana Engineer](https://altinity.com/blog/2020-5-20-reducing-clickhouse-storage-cost-with-the-low-cardinality-type-lessons-from-an-instana-engineer)
-- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/ClickHouse/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf)
- Blog: [Optimizing ClickHouse with Schemas and Codecs](https://clickhouse.com/blog/optimize-clickhouse-codecs-compression-schema)
- Blog: [Working with time series data in ClickHouse](https://clickhouse.com/blog/working-with-time-series-data-and-functions-ClickHouse)
+- [String Optimization (video presentation in Russian)](https://youtu.be/rqf-ILRgBdY?list=PL0Z2YDlm0b3iwXCpEFiOOYmwXzVmjJfEt). [Slides in English](https://github.com/ClickHouse/clickhouse-presentations/raw/master/meetup19/string_optimization.pdf)
diff --git a/docs/en/sql-reference/functions/uuid-functions.md b/docs/en/sql-reference/functions/uuid-functions.md
index e990023efbc..0bde207dcc9 100644
--- a/docs/en/sql-reference/functions/uuid-functions.md
+++ b/docs/en/sql-reference/functions/uuid-functions.md
@@ -567,12 +567,13 @@ While no standard or recommendation exists for the epoch of Snowflake IDs, imple
**Syntax**
``` sql
-generateSnowflakeID([expr])
+generateSnowflakeID([expr, [machine_id]])
```
**Arguments**
- `expr` — An arbitrary [expression](../../sql-reference/syntax.md#syntax-expressions) used to bypass [common subexpression elimination](../../sql-reference/functions/index.md#common-subexpression-elimination) if the function is called multiple times in a query. The value of the expression has no effect on the returned Snowflake ID. Optional.
+- `machine_id` — A machine ID, the lowest 10 bits are used. [Int64](../data-types/int-uint.md). Optional.
**Returned value**
@@ -608,6 +609,16 @@ SELECT generateSnowflakeID(1), generateSnowflakeID(2);
└────────────────────────┴────────────────────────┘
```
+**Example with expression and a machine ID**
+
+```
+SELECT generateSnowflakeID('expr', 1);
+
+┌─generateSnowflakeID('expr', 1)─┐
+│ 7201148511606784002 │
+└────────────────────────────────┘
+```
+
## snowflakeToDateTime
:::warning
diff --git a/docs/zh/changelog/index.md b/docs/zh/changelog/index.md
index c91d8bcf4d1..cd77a8c03cf 100644
--- a/docs/zh/changelog/index.md
+++ b/docs/zh/changelog/index.md
@@ -356,7 +356,7 @@ sidebar_label: "\u53D8\u66F4\u65E5\u5FD7"
#### 新功能 {#new-feature-1}
-- 添加 `deduplicate_blocks_in_dependent_materialized_views` 用于控制具有实例化视图的表中幂等插入的行为的选项。 这个新功能是由Altinity的特殊要求添加到错误修正版本中的。
+- 添加 `deduplicate_blocks_in_dependent_materialized_views` 用于控制具有实例化视图的表中幂等插入的行为的选项。
[#9070](https://github.com/ClickHouse/ClickHouse/pull/9070) [(urykhy)](https://github.com/urykhy)
### ClickHouse版本v20.1.2.4,2020-01-22 {#clickhouse-release-v20-1-2-4-2020-01-22}
diff --git a/src/Columns/ColumnObject.cpp b/src/Columns/ColumnObject.cpp
index adcd42b16e9..9c9dade3dd8 100644
--- a/src/Columns/ColumnObject.cpp
+++ b/src/Columns/ColumnObject.cpp
@@ -1101,4 +1101,10 @@ void ColumnObject::finalize()
checkObjectHasNoAmbiguosPaths(getKeys());
}
+void ColumnObject::updateHashFast(SipHash & hash) const
+{
+ for (const auto & entry : subcolumns)
+ for (auto & part : entry->data.data)
+ part->updateHashFast(hash);
+}
}
diff --git a/src/Columns/ColumnObject.h b/src/Columns/ColumnObject.h
index fadf2e18779..7470dfa6302 100644
--- a/src/Columns/ColumnObject.h
+++ b/src/Columns/ColumnObject.h
@@ -253,7 +253,7 @@ public:
const char * skipSerializedInArena(const char *) const override { throwMustBeConcrete(); }
void updateHashWithValue(size_t, SipHash &) const override { throwMustBeConcrete(); }
void updateWeakHash32(WeakHash32 &) const override { throwMustBeConcrete(); }
- void updateHashFast(SipHash &) const override { throwMustBeConcrete(); }
+ void updateHashFast(SipHash & hash) const override;
void expand(const Filter &, bool) override { throwMustBeConcrete(); }
bool hasEqualValues() const override { throwMustBeConcrete(); }
size_t byteSizeAt(size_t) const override { throwMustBeConcrete(); }
diff --git a/src/Common/CollectionOfDerived.h b/src/Common/CollectionOfDerived.h
new file mode 100644
index 00000000000..97c0c3fbc06
--- /dev/null
+++ b/src/Common/CollectionOfDerived.h
@@ -0,0 +1,184 @@
+#pragma once
+
+#include
+
+#include
+
+#include
+#include
+#include
+#include
+#include
+
+
+namespace DB
+{
+
+namespace ErrorCodes
+{
+ extern const int LOGICAL_ERROR;
+}
+
+/* This is a collections of objects derived from ItemBase.
+* Collection contains no more than one instance for each derived type.
+* The derived type is used to access the instance.
+*/
+
+template
+class CollectionOfDerivedItems
+{
+public:
+ using Self = CollectionOfDerivedItems;
+ using ItemPtr = std::shared_ptr;
+
+private:
+ struct Rec
+ {
+ std::type_index type_idx;
+ ItemPtr ptr;
+
+ bool operator<(const Rec & other) const
+ {
+ return type_idx < other.type_idx;
+ }
+
+ bool operator<(const std::type_index & value) const
+ {
+ return type_idx < value;
+ }
+
+ bool operator==(const Rec & other) const
+ {
+ return type_idx == other.type_idx;
+ }
+ };
+ using Records = std::vector;
+
+public:
+ void swap(Self & other) noexcept
+ {
+ records.swap(other.records);
+ }
+
+ void clear()
+ {
+ records.clear();
+ }
+
+ bool empty() const
+ {
+ return records.empty();
+ }
+
+ size_t size() const
+ {
+ return records.size();
+ }
+
+ Self clone() const
+ {
+ Self result;
+ result.records.reserve(records.size());
+ for (const auto & rec : records)
+ result.records.emplace_back(rec.type_idx, rec.ptr->clone());
+ return result;
+ }
+
+ void append(Self && other)
+ {
+ auto middle_idx = records.size();
+ std::move(other.records.begin(), other.records.end(), std::back_inserter(records));
+ std::inplace_merge(records.begin(), records.begin() + middle_idx, records.end());
+ chassert(isUniqTypes());
+ }
+
+ template
+ void add(std::shared_ptr info)
+ {
+ static_assert(std::is_base_of_v, "Template parameter must inherit items base class");
+ return addImpl(std::type_index(typeid(T)), std::move(info));
+ }
+
+ template
+ std::shared_ptr get() const
+ {
+ static_assert(std::is_base_of_v, "Template parameter must inherit items base class");
+ auto it = getImpl(std::type_index(typeid(T)));
+ if (it == records.cend())
+ return nullptr;
+ auto cast = std::dynamic_pointer_cast(it->ptr);
+ chassert(cast);
+ return cast;
+ }
+
+ template
+ std::shared_ptr extract()
+ {
+ static_assert(std::is_base_of_v, "Template parameter must inherit items base class");
+ auto it = getImpl(std::type_index(typeid(T)));
+ if (it == records.cend())
+ return nullptr;
+ auto cast = std::dynamic_pointer_cast(it->ptr);
+ chassert(cast);
+
+ records.erase(it);
+ return cast;
+ }
+
+ std::string debug() const
+ {
+ std::string result;
+
+ for (auto & rec : records)
+ {
+ result.append(rec.type_idx.name());
+ result.append(" ");
+ }
+
+ return result;
+ }
+
+private:
+ bool isUniqTypes() const
+ {
+ auto uniq_it = std::adjacent_find(records.begin(), records.end());
+
+ return uniq_it == records.end();
+ }
+
+ void addImpl(std::type_index type_idx, ItemPtr item)
+ {
+ auto it = std::lower_bound(records.begin(), records.end(), type_idx);
+
+ if (it == records.end())
+ {
+ records.emplace_back(type_idx, item);
+ return;
+ }
+
+ if (it->type_idx == type_idx)
+ throw Exception(ErrorCodes::LOGICAL_ERROR, "inserted items must be unique by their type, type {} is inserted twice", type_idx.name());
+
+
+ records.emplace(it, type_idx, item);
+
+ chassert(isUniqTypes());
+ }
+
+ Records::const_iterator getImpl(std::type_index type_idx) const
+ {
+ auto it = std::lower_bound(records.cbegin(), records.cend(), type_idx);
+
+ if (it == records.cend())
+ return records.cend();
+
+ if (it->type_idx != type_idx)
+ return records.cend();
+
+ return it;
+ }
+
+ Records records;
+};
+
+}
diff --git a/src/Core/Settings.h b/src/Core/Settings.h
index 92b1c750a55..f19f2827b62 100644
--- a/src/Core/Settings.h
+++ b/src/Core/Settings.h
@@ -36,7 +36,7 @@ class IColumn;
M(Dialect, dialect, Dialect::clickhouse, "Which dialect will be used to parse query", 0)\
M(UInt64, min_compress_block_size, 65536, "The actual size of the block to compress, if the uncompressed data less than max_compress_block_size is no less than this value and no less than the volume of data for one mark.", 0) \
M(UInt64, max_compress_block_size, 1048576, "The maximum size of blocks of uncompressed data before compressing for writing to a table.", 0) \
- M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size for reading", 0) \
+ M(UInt64, max_block_size, DEFAULT_BLOCK_SIZE, "Maximum block size in rows for reading", 0) \
M(UInt64, max_insert_block_size, DEFAULT_INSERT_BLOCK_SIZE, "The maximum block size for insertion, if we control the creation of blocks for insertion.", 0) \
M(UInt64, min_insert_block_size_rows, DEFAULT_INSERT_BLOCK_SIZE, "Squash blocks passed to INSERT query to specified size in rows, if blocks are not big enough.", 0) \
M(UInt64, min_insert_block_size_bytes, (DEFAULT_INSERT_BLOCK_SIZE * 256), "Squash blocks passed to INSERT query to specified size in bytes, if blocks are not big enough.", 0) \
@@ -609,9 +609,8 @@ class IColumn;
M(Bool, optimize_time_filter_with_preimage, true, "Optimize Date and DateTime predicates by converting functions into equivalent comparisons without conversions (e.g. toYear(col) = 2023 -> col >= '2023-01-01' AND col <= '2023-12-31')", 0) \
M(Bool, normalize_function_names, true, "Normalize function names to their canonical names", 0) \
M(Bool, enable_early_constant_folding, true, "Enable query optimization where we analyze function and subqueries results and rewrite query if there are constants there", 0) \
- M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views if the block is not a duplicate for the table. Use true to always deduplicate in dependent tables.", 0) \
+ M(Bool, deduplicate_blocks_in_dependent_materialized_views, false, "Should deduplicate blocks for materialized views. Use true to always deduplicate in dependent tables.", 0) \
M(Bool, throw_if_deduplication_in_dependent_materialized_views_enabled_with_async_insert, true, "Throw exception on INSERT query when the setting `deduplicate_blocks_in_dependent_materialized_views` is enabled along with `async_insert`. It guarantees correctness, because these features can't work together.", 0) \
- M(Bool, update_insert_deduplication_token_in_dependent_materialized_views, false, "Should update insert deduplication token with table identifier during insert in dependent materialized views.", 0) \
M(Bool, materialized_views_ignore_errors, false, "Allows to ignore errors for MATERIALIZED VIEW, and deliver original block to the table regardless of MVs", 0) \
M(Bool, ignore_materialized_views_with_dropped_target_table, false, "Ignore MVs with dropped target table during pushing to views", 0) \
M(Bool, use_compact_format_in_distributed_parts_names, true, "Changes format of directories names for distributed table insert parts.", 0) \
@@ -977,6 +976,7 @@ class IColumn;
#define OBSOLETE_SETTINGS(M, ALIAS) \
/** Obsolete settings that do nothing but left for compatibility reasons. Remove each one after half a year of obsolescence. */ \
+ MAKE_OBSOLETE(M, Bool, update_insert_deduplication_token_in_dependent_materialized_views, 0) \
MAKE_OBSOLETE(M, UInt64, max_memory_usage_for_all_queries, 0) \
MAKE_OBSOLETE(M, UInt64, multiple_joins_rewriter_version, 0) \
MAKE_OBSOLETE(M, Bool, enable_debug_queries, false) \
diff --git a/src/Daemon/BaseDaemon.cpp b/src/Daemon/BaseDaemon.cpp
index 9b132c3e199..4bc8d436df7 100644
--- a/src/Daemon/BaseDaemon.cpp
+++ b/src/Daemon/BaseDaemon.cpp
@@ -1306,6 +1306,10 @@ void BaseDaemon::setupWatchdog()
int status = 0;
do
{
+ // Close log files to prevent keeping descriptors of unlinked rotated files.
+ // On next log write files will be reopened.
+ closeLogs(logger());
+
if (-1 != waitpid(pid, &status, WUNTRACED | WCONTINUED) || errno == ECHILD)
{
if (WIFSTOPPED(status))
diff --git a/src/Databases/DatabaseAtomic.cpp b/src/Databases/DatabaseAtomic.cpp
index 9e7004e72a9..d86e29ca915 100644
--- a/src/Databases/DatabaseAtomic.cpp
+++ b/src/Databases/DatabaseAtomic.cpp
@@ -39,8 +39,10 @@ namespace ErrorCodes
class AtomicDatabaseTablesSnapshotIterator final : public DatabaseTablesSnapshotIterator
{
public:
- explicit AtomicDatabaseTablesSnapshotIterator(DatabaseTablesSnapshotIterator && base)
- : DatabaseTablesSnapshotIterator(std::move(base)) {}
+ explicit AtomicDatabaseTablesSnapshotIterator(DatabaseTablesSnapshotIterator && base) noexcept
+ : DatabaseTablesSnapshotIterator(std::move(base))
+ {
+ }
UUID uuid() const override { return table()->getStorageID().uuid; }
};
@@ -111,12 +113,12 @@ StoragePtr DatabaseAtomic::detachTable(ContextPtr /* context */, const String &
// it is important to call the destructors of not_in_use without
// locked mutex to avoid potential deadlock.
DetachedTables not_in_use;
- StoragePtr table;
+ StoragePtr detached_table;
{
std::lock_guard lock(mutex);
- table = DatabaseOrdinary::detachTableUnlocked(name);
+ detached_table = DatabaseOrdinary::detachTableUnlocked(name);
table_name_to_path.erase(name);
- detached_tables.emplace(table->getStorageID().uuid, table);
+ detached_tables.emplace(detached_table->getStorageID().uuid, detached_table);
not_in_use = cleanupDetachedTables();
}
@@ -126,7 +128,7 @@ StoragePtr DatabaseAtomic::detachTable(ContextPtr /* context */, const String &
LOG_DEBUG(log, "Finished removing not used detached tables");
}
- return table;
+ return detached_table;
}
void DatabaseAtomic::dropTable(ContextPtr local_context, const String & table_name, bool sync)
diff --git a/src/Databases/DatabaseAtomic.h b/src/Databases/DatabaseAtomic.h
index b59edd479ba..4a4ccfa2573 100644
--- a/src/Databases/DatabaseAtomic.h
+++ b/src/Databases/DatabaseAtomic.h
@@ -1,7 +1,8 @@
#pragma once
-#include
#include
+#include
+#include
namespace DB
diff --git a/src/Databases/DatabaseLazy.cpp b/src/Databases/DatabaseLazy.cpp
index da942cebf8f..5017c9b25cb 100644
--- a/src/Databases/DatabaseLazy.cpp
+++ b/src/Databases/DatabaseLazy.cpp
@@ -188,6 +188,13 @@ void DatabaseLazy::attachTable(ContextPtr /* context_ */, const String & table_n
it->second.expiration_iterator = cache_expiration_queue.emplace(cache_expiration_queue.end(), current_time, table_name);
+ LOG_DEBUG(log, "Add info for detached table {} to snapshot.", backQuote(table_name));
+ if (snapshot_detached_tables.contains(table_name))
+ {
+ LOG_DEBUG(log, "Clean info about detached table {} from snapshot.", backQuote(table_name));
+ snapshot_detached_tables.erase(table_name);
+ }
+
CurrentMetrics::add(CurrentMetrics::AttachedTable, 1);
}
@@ -204,6 +211,15 @@ StoragePtr DatabaseLazy::detachTable(ContextPtr /* context */, const String & ta
if (it->second.expiration_iterator != cache_expiration_queue.end())
cache_expiration_queue.erase(it->second.expiration_iterator);
tables_cache.erase(it);
+ LOG_DEBUG(log, "Add info for detached table {} to snapshot.", backQuote(table_name));
+ snapshot_detached_tables.emplace(
+ table_name,
+ SnapshotDetachedTable{
+ .database = res->getStorageID().database_name,
+ .table = res->getStorageID().table_name,
+ .uuid = res->getStorageID().uuid,
+ .metadata_path = getObjectMetadataPath(table_name),
+ .is_permanently = false});
CurrentMetrics::sub(CurrentMetrics::AttachedTable, 1);
}
diff --git a/src/Databases/DatabaseOnDisk.cpp b/src/Databases/DatabaseOnDisk.cpp
index 07a250e72c7..42730cfd53d 100644
--- a/src/Databases/DatabaseOnDisk.cpp
+++ b/src/Databases/DatabaseOnDisk.cpp
@@ -23,6 +23,7 @@
#include
#include
#include
+#include
#include
#include
#include
@@ -308,6 +309,16 @@ void DatabaseOnDisk::detachTablePermanently(ContextPtr query_context, const Stri
try
{
FS::createFile(detached_permanently_flag);
+
+ std::lock_guard lock(mutex);
+ if (const auto it = snapshot_detached_tables.find(table_name); it == snapshot_detached_tables.end())
+ {
+ throw Exception(ErrorCodes::LOGICAL_ERROR, "Snapshot doesn't contain info about detached table={}", table_name);
+ }
+ else
+ {
+ it->second.is_permanently = true;
+ }
}
catch (Exception & e)
{
diff --git a/src/Databases/DatabaseOrdinary.cpp b/src/Databases/DatabaseOrdinary.cpp
index 6555c4444e2..3ab5d3fa697 100644
--- a/src/Databases/DatabaseOrdinary.cpp
+++ b/src/Databases/DatabaseOrdinary.cpp
@@ -189,7 +189,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
size_t prev_tables_count = metadata.parsed_tables.size();
size_t prev_total_dictionaries = metadata.total_dictionaries;
- auto process_metadata = [&metadata, is_startup, this](const String & file_name)
+ auto process_metadata = [&metadata, is_startup, local_context, this](const String & file_name)
{
fs::path path(getMetadataPath());
fs::path file_path(file_name);
@@ -197,7 +197,7 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
try
{
- auto ast = parseQueryFromMetadata(log, getContext(), full_path.string(), /*throw_on_error*/ true, /*remove_empty*/ false);
+ auto ast = parseQueryFromMetadata(log, local_context, full_path.string(), /*throw_on_error*/ true, /*remove_empty*/ false);
if (ast)
{
FunctionNameNormalizer::visit(ast.get());
@@ -226,8 +226,23 @@ void DatabaseOrdinary::loadTablesMetadata(ContextPtr local_context, ParsedTables
if (fs::exists(full_path.string() + detached_suffix))
{
const std::string table_name = unescapeForFileName(file_name.substr(0, file_name.size() - 4));
- permanently_detached_tables.push_back(table_name);
LOG_DEBUG(log, "Skipping permanently detached table {}.", backQuote(table_name));
+
+ std::lock_guard lock(mutex);
+ permanently_detached_tables.push_back(table_name);
+
+ const auto detached_table_name = create_query->getTable();
+
+ snapshot_detached_tables.emplace(
+ detached_table_name,
+ SnapshotDetachedTable{
+ .database = create_query->getDatabase(),
+ .table = detached_table_name,
+ .uuid = create_query->uuid,
+ .metadata_path = getObjectMetadataPath(detached_table_name),
+ .is_permanently = true});
+
+ LOG_TRACE(log, "Add permanently detached table {} to system.detached_tables", detached_table_name);
return;
}
@@ -489,6 +504,12 @@ DatabaseTablesIteratorPtr DatabaseOrdinary::getTablesIterator(ContextPtr local_c
return DatabaseWithOwnTablesBase::getTablesIterator(local_context, filter_by_table_name, skip_not_loaded);
}
+DatabaseDetachedTablesSnapshotIteratorPtr DatabaseOrdinary::getDetachedTablesIterator(
+ ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const
+{
+ return DatabaseWithOwnTablesBase::getDetachedTablesIterator(local_context, filter_by_table_name, skip_not_loaded);
+}
+
Strings DatabaseOrdinary::getAllTableNames(ContextPtr) const
{
std::set unique_names;
diff --git a/src/Databases/DatabaseOrdinary.h b/src/Databases/DatabaseOrdinary.h
index ef00ac8fdfa..c2c5775e5ab 100644
--- a/src/Databases/DatabaseOrdinary.h
+++ b/src/Databases/DatabaseOrdinary.h
@@ -57,6 +57,9 @@ public:
LoadTaskPtr startupDatabaseAsync(AsyncLoader & async_loader, LoadJobSet startup_after, LoadingStrictnessLevel mode) override;
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
+ DatabaseDetachedTablesSnapshotIteratorPtr getDetachedTablesIterator(
+ ContextPtr local_context, const DatabaseOnDisk::FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
+
Strings getAllTableNames(ContextPtr context) const override;
void alterTable(
@@ -64,7 +67,11 @@ public:
const StorageID & table_id,
const StorageInMemoryMetadata & metadata) override;
- Strings getNamesOfPermanentlyDetachedTables() const override { return permanently_detached_tables; }
+ Strings getNamesOfPermanentlyDetachedTables() const override
+ {
+ std::lock_guard lock(mutex);
+ return permanently_detached_tables;
+ }
protected:
virtual void commitAlterTable(
@@ -74,7 +81,7 @@ protected:
const String & statement,
ContextPtr query_context);
- Strings permanently_detached_tables;
+ Strings permanently_detached_tables TSA_GUARDED_BY(mutex);
std::unordered_map load_table TSA_GUARDED_BY(mutex);
std::unordered_map startup_table TSA_GUARDED_BY(mutex);
diff --git a/src/Databases/DatabasesCommon.cpp b/src/Databases/DatabasesCommon.cpp
index 6426123bb4f..fe0baf30e57 100644
--- a/src/Databases/DatabasesCommon.cpp
+++ b/src/Databases/DatabasesCommon.cpp
@@ -2,12 +2,9 @@
#include
#include
-#include
-#include
-#include
-#include
#include
#include
+#include
#include
#include
#include
@@ -16,6 +13,10 @@
#include
#include
#include
+#include
+#include
+#include
+#include
namespace DB
@@ -237,6 +238,24 @@ DatabaseTablesIteratorPtr DatabaseWithOwnTablesBase::getTablesIterator(ContextPt
return std::make_unique(std::move(filtered_tables), database_name);
}
+DatabaseDetachedTablesSnapshotIteratorPtr DatabaseWithOwnTablesBase::getDetachedTablesIterator(
+ ContextPtr, const FilterByNameFunction & filter_by_table_name, bool /* skip_not_loaded */) const
+{
+ std::lock_guard lock(mutex);
+ if (!filter_by_table_name)
+ return std::make_unique(snapshot_detached_tables);
+
+ SnapshotDetachedTables filtered_detached_tables;
+ for (const auto & [detached_table_name, snapshot] : snapshot_detached_tables)
+ if (filter_by_table_name(detached_table_name))
+ {
+ filtered_detached_tables.emplace(detached_table_name, snapshot);
+ }
+
+
+ return std::make_unique(std::move(filtered_detached_tables));
+}
+
bool DatabaseWithOwnTablesBase::empty() const
{
std::lock_guard lock(mutex);
@@ -251,27 +270,36 @@ StoragePtr DatabaseWithOwnTablesBase::detachTable(ContextPtr /* context_ */, con
StoragePtr DatabaseWithOwnTablesBase::detachTableUnlocked(const String & table_name)
{
- StoragePtr res;
-
auto it = tables.find(table_name);
if (it == tables.end())
throw Exception(ErrorCodes::UNKNOWN_TABLE, "Table {}.{} doesn't exist",
backQuote(database_name), backQuote(table_name));
- res = it->second;
+
+ auto table_storage = it->second;
+
+ snapshot_detached_tables.emplace(
+ table_name,
+ SnapshotDetachedTable{
+ .database = it->second->getStorageID().getDatabaseName(),
+ .table = table_name,
+ .uuid = it->second->getStorageID().uuid,
+ .metadata_path = getObjectMetadataPath(table_name),
+ .is_permanently = false});
+
tables.erase(it);
- res->is_detached = true;
+ table_storage->is_detached = true;
- if (res->isSystemStorage() == false)
- CurrentMetrics::sub(getAttachedCounterForStorage(res), 1);
+ if (table_storage->isSystemStorage() == false)
+ CurrentMetrics::sub(getAttachedCounterForStorage(table_storage), 1);
- auto table_id = res->getStorageID();
+ auto table_id = table_storage->getStorageID();
if (table_id.hasUUID())
{
assert(database_name == DatabaseCatalog::TEMPORARY_DATABASE || getUUID() != UUIDHelpers::Nil);
DatabaseCatalog::instance().removeUUIDMapping(table_id.uuid);
}
- return res;
+ return table_storage;
}
void DatabaseWithOwnTablesBase::attachTable(ContextPtr /* context_ */, const String & table_name, const StoragePtr & table, const String &)
@@ -300,6 +328,8 @@ void DatabaseWithOwnTablesBase::attachTableUnlocked(const String & table_name, c
throw Exception(ErrorCodes::TABLE_ALREADY_EXISTS, "Table {} already exists.", table_id.getFullTableName());
}
+ snapshot_detached_tables.erase(table_name);
+
/// It is important to reset is_detached here since in case of RENAME in
/// non-Atomic database the is_detached is set to true before RENAME.
table->is_detached = false;
@@ -337,6 +367,7 @@ void DatabaseWithOwnTablesBase::shutdown()
std::lock_guard lock(mutex);
tables.clear();
+ snapshot_detached_tables.clear();
}
DatabaseWithOwnTablesBase::~DatabaseWithOwnTablesBase()
diff --git a/src/Databases/DatabasesCommon.h b/src/Databases/DatabasesCommon.h
index 2eecf8a564f..1ca49e90c23 100644
--- a/src/Databases/DatabasesCommon.h
+++ b/src/Databases/DatabasesCommon.h
@@ -37,6 +37,9 @@ public:
DatabaseTablesIteratorPtr getTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
+ DatabaseDetachedTablesSnapshotIteratorPtr
+ getDetachedTablesIterator(ContextPtr context, const FilterByNameFunction & filter_by_table_name, bool skip_not_loaded) const override;
+
std::vector> getTablesForBackup(const FilterByNameFunction & filter, const ContextPtr & local_context) const override;
void createTableRestoredFromBackup(const ASTPtr & create_table_query, ContextMutablePtr local_context, std::shared_ptr restore_coordination, UInt64 timeout_ms) override;
@@ -46,12 +49,13 @@ public:
protected:
Tables tables TSA_GUARDED_BY(mutex);
+ SnapshotDetachedTables snapshot_detached_tables TSA_GUARDED_BY(mutex);
LoggerPtr log;
DatabaseWithOwnTablesBase(const String & name_, const String & logger, ContextPtr context);
void attachTableUnlocked(const String & table_name, const StoragePtr & table) TSA_REQUIRES(mutex);
- StoragePtr detachTableUnlocked(const String & table_name) TSA_REQUIRES(mutex);
+ StoragePtr detachTableUnlocked(const String & table_name) TSA_REQUIRES(mutex);
StoragePtr getTableUnlocked(const String & table_name) const TSA_REQUIRES(mutex);
StoragePtr tryGetTableNoWait(const String & table_name) const;
};
diff --git a/src/Databases/IDatabase.h b/src/Databases/IDatabase.h
index b00f2fe4baf..ce5a52b1b0f 100644
--- a/src/Databases/IDatabase.h
+++ b/src/Databases/IDatabase.h
@@ -5,20 +5,22 @@
#include
#include
#include
+#include
+#include
#include
#include
-#include
#include
+#include
#include
#include
-#include
#include
#include
+#include