diff --git a/src/Storages/ColumnsDescription.cpp b/src/Storages/ColumnsDescription.cpp
index 7a43ae7af4b..1a90312e076 100644
--- a/src/Storages/ColumnsDescription.cpp
+++ b/src/Storages/ColumnsDescription.cpp
@@ -173,16 +173,16 @@ static auto getNameRange(const ColumnsDescription::ColumnsContainer & columns, c
 {
     String name_with_dot = name_without_dot + ".";
 
-    auto begin = columns.begin();
-    for (; begin != columns.end(); ++begin)
+    /// First we need to check if we have column with name name_without_dot
+    /// and if not - check if we have names that start with name_with_dot
+    for (auto it = columns.begin(); it != columns.end(); ++it)
     {
-        if (begin->name == name_without_dot)
-            return std::make_pair(begin, std::next(begin));
-
-        if (startsWith(begin->name, name_with_dot))
-            break;
+        if (it->name == name_without_dot)
+            return std::make_pair(it, std::next(it));
     }
 
+    auto begin = std::find_if(columns.begin(), columns.end(), [&](const auto & column){ return startsWith(column.name, name_with_dot); });
+
     if (begin == columns.end())
         return std::make_pair(begin, begin);
 
diff --git a/tests/queries/0_stateless/02320_alter_columns_with_dots.reference b/tests/queries/0_stateless/02320_alter_columns_with_dots.reference
new file mode 100644
index 00000000000..3c3e828924e
--- /dev/null
+++ b/tests/queries/0_stateless/02320_alter_columns_with_dots.reference
@@ -0,0 +1,27 @@
+id	String					
+abc.1	String					
+abc.2	String					
+abc	String					
+id	String					
+abc.2	String					
+abc	String					
+abc.1	String					
+id	String					
+abc	String					
+abc.2	String					
+abc.1	String					
+id	String					
+abc.2	String					
+abc	String					
+abc.1	String					
+id	String					
+abc	String					
+abc.2	String					
+abc.1	String					
+id	String					
+abc.2	String					
+abc.1	String					
+abc	String					
+id	String					
+abc.2	String					
+abc.1	String					
diff --git a/tests/queries/0_stateless/02320_alter_columns_with_dots.sql b/tests/queries/0_stateless/02320_alter_columns_with_dots.sql
new file mode 100644
index 00000000000..1b48538f987
--- /dev/null
+++ b/tests/queries/0_stateless/02320_alter_columns_with_dots.sql
@@ -0,0 +1,15 @@
+DROP TABLE IF EXISTS test;
+CREATE TABLE test (id String, `abc.1` String, `abc.2` String, `abc` String) ENGINE MergeTree order by id;
+DESC TABLE test;
+ALTER TABLE test MODIFY COLUMN `abc.1` String AFTER `abc`;
+DESC TABLE test;
+ALTER TABLE test MODIFY COLUMN `abc.2` String AFTER `abc`;
+DESC TABLE test;
+ALTER TABLE test MODIFY COLUMN `abc` String AFTER `abc.2`;
+DESC TABLE test;
+ALTER TABLE test MODIFY COLUMN `abc` String AFTER `id`;
+DESC TABLE test;
+ALTER TABLE test MODIFY COLUMN `abc` String AFTER `abc.1`;
+DESC TABLE test;
+ALTER TABLE test DROP COLUMN `abc`;
+DESC TABLE test;