diff --git a/src/Interpreters/Cache/FileSegment.cpp b/src/Interpreters/Cache/FileSegment.cpp
index 34de2c71bde..77749ed3233 100644
--- a/src/Interpreters/Cache/FileSegment.cpp
+++ b/src/Interpreters/Cache/FileSegment.cpp
@@ -538,7 +538,8 @@ void FileSegment::setBroken()
     assertNotDetachedUnlocked(lock);
     assertIsDownloaderUnlocked("setBroken", lock);
 
-    resetDownloadingStateUnlocked(lock);
+    if (download_state == State::DOWNLOADING)
+        resetDownloadingStateUnlocked(lock);
     if (download_state != State::DOWNLOADED)
         download_state = State::PARTIALLY_DOWNLOADED_NO_CONTINUATION;