diff --git a/src/Databases/DatabaseMemory.cpp b/src/Databases/DatabaseMemory.cpp
index c0af027e027..0ca1de09dd1 100644
--- a/src/Databases/DatabaseMemory.cpp
+++ b/src/Databases/DatabaseMemory.cpp
@@ -42,12 +42,17 @@ void DatabaseMemory::dropTable(
     try
     {
         table->drop();
-        fs::path table_data_dir{getTableDataPath(table_name)};
-        if (fs::exists(table_data_dir))
-            fs::remove_all(table_data_dir);
+        if (table->storesDataOnDisk())
+        {
+            assert(database_name != DatabaseCatalog::TEMPORARY_DATABASE);
+            fs::path table_data_dir{getTableDataPath(table_name)};
+            if (fs::exists(table_data_dir))
+                fs::remove_all(table_data_dir);
+        }
     }
     catch (...)
     {
+        assert(database_name != DatabaseCatalog::TEMPORARY_DATABASE);
         attachTableUnlocked(table_name, table, lock);
         throw;
     }