diff --git a/src/Interpreters/PreparedSets.cpp b/src/Interpreters/PreparedSets.cpp
index ea8d9a62b8b..9f646825d9f 100644
--- a/src/Interpreters/PreparedSets.cpp
+++ b/src/Interpreters/PreparedSets.cpp
@@ -189,7 +189,8 @@ SetPtr FutureSetFromSubquery::buildOrderedSetInplace(const ContextPtr & context)
         }
     }
 
-    set_and_key->set->fillSetElements();
+    if (!set_and_key->set->hasSetElements())
+        set_and_key->set->fillSetElements();
 
     return buildSetInplace(context);
 }
diff --git a/src/Interpreters/Set.h b/src/Interpreters/Set.h
index 7136b090c42..7e8e0f2371b 100644
--- a/src/Interpreters/Set.h
+++ b/src/Interpreters/Set.h
@@ -77,6 +77,7 @@ public:
     const DataTypes & getElementsTypes() const { return set_elements_types; }
 
     bool hasExplicitSetElements() const { return fill_set_elements || (!set_elements.empty() && set_elements.front()->size() == data.getTotalRowCount()); }
+    bool hasSetElements() const { return !set_elements.empty(); }
     Columns getSetElements() const { checkIsCreated(); return { set_elements.begin(), set_elements.end() }; }
 
     void checkColumnsNumber(size_t num_key_columns) const;