diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp
index 9b391324455..2713b7cb35f 100644
--- a/src/Formats/FormatFactory.cpp
+++ b/src/Formats/FormatFactory.cpp
@@ -410,7 +410,7 @@ std::unique_ptr<ReadBuffer> FormatFactory::wrapReadBufferIfNeeded(
 
 static void addExistingProgressToOutputFormat(OutputFormatPtr format, ContextPtr context)
 {
-    auto element_id = context->getProcessListElement();
+    auto element_id = context->getProcessListElementSafe();
     if (element_id)
     {
         /// While preparing the query there might have been progress (for example in subscalar subqueries) so add it here
diff --git a/src/Interpreters/executeQuery.cpp b/src/Interpreters/executeQuery.cpp
index 22230c53698..861332c65fd 100644
--- a/src/Interpreters/executeQuery.cpp
+++ b/src/Interpreters/executeQuery.cpp
@@ -1336,7 +1336,46 @@ void executeQuery(
     BlockIO streams;
     OutputFormatPtr output_format;
 
-    std::tie(ast, streams) = executeQueryImpl(begin, end, context, false, QueryProcessingStage::Complete, &istr);
+    auto update_format_for_exception_if_needed = [&]()
+    {
+        if (!output_format)
+        {
+            try
+            {
+                String format_name = context->getDefaultFormat();
+                output_format = FormatFactory::instance().getOutputFormat(format_name, ostr, {}, context, output_format_settings);
+                if (output_format && output_format->supportsWritingException())
+                {
+                    /// Force an update of the headers before we start writing
+                    result_details.content_type = output_format->getContentType();
+                    result_details.format = format_name;
+                    set_result_details(result_details);
+                    set_result_details = nullptr;
+                }
+            }
+            catch (const DB::Exception & e)
+            {
+                /// Ignore this exception and report the original one
+                LOG_WARNING(&Poco::Logger::get("executeQuery"), getExceptionMessageAndPattern(e, true));
+            }
+        }
+    };
+
+    try
+    {
+        std::tie(ast, streams) = executeQueryImpl(begin, end, context, false, QueryProcessingStage::Complete, &istr);
+    }
+    catch (...)
+    {
+        if (handle_exception_in_output_format)
+        {
+            update_format_for_exception_if_needed();
+            if (output_format)
+                handle_exception_in_output_format(*output_format);
+        }
+        throw;
+    }
+
     auto & pipeline = streams.pipeline;
 
     std::unique_ptr<WriteBuffer> compressed_buffer;
@@ -1430,8 +1469,12 @@ void executeQuery(
     }
     catch (...)
     {
-        if (handle_exception_in_output_format && output_format)
-            handle_exception_in_output_format(*output_format);
+        if (handle_exception_in_output_format)
+        {
+            update_format_for_exception_if_needed();
+            if (output_format)
+                handle_exception_in_output_format(*output_format);
+        }
         streams.onException();
         throw;
     }
diff --git a/tests/queries/0_stateless/02899_use_default_format_on_http_exception.reference b/tests/queries/0_stateless/02899_use_default_format_on_http_exception.reference
new file mode 100644
index 00000000000..a943df06764
--- /dev/null
+++ b/tests/queries/0_stateless/02899_use_default_format_on_http_exception.reference
@@ -0,0 +1,25 @@
+INSERT WITH default_format=JSON
+Content-Type:application/json;charset=UTF-8
+"exception":"Code:62.
+
+INSERT WITH default_format=XML
+Content-Type:application/xml;charset=UTF-8
+<exception>Code:62.DB::Ex---tion:
+
+INSERT WITH default_format=BADFORMAT
+Content-Type:text/plain;charset=UTF-8
+X-ClickHouse-Ex---tion-Code:62
+Code:62.DB::Ex---tion:
+
+INSERT WITH X-ClickHouse-Format: JSON
+Content-Type:application/json;charset=UTF-8
+"exception":"Code:62.
+
+INSERT WITH X-ClickHouse-Format: XML
+Content-Type:application/xml;charset=UTF-8
+<exception>Code:62.DB::Ex---tion:
+
+INSERT WITH X-ClickHouse-Format: BADFORMAT
+Content-Type:text/plain;charset=UTF-8
+X-ClickHouse-Ex---tion-Code:62
+Code:62.DB::Ex---tion:
diff --git a/tests/queries/0_stateless/02899_use_default_format_on_http_exception.sh b/tests/queries/0_stateless/02899_use_default_format_on_http_exception.sh
new file mode 100755
index 00000000000..f92ab7db4fb
--- /dev/null
+++ b/tests/queries/0_stateless/02899_use_default_format_on_http_exception.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+CH_URL="$CLICKHOUSE_URL&http_write_exception_in_output_format=1"
+
+echo "INSERT WITH default_format=JSON"
+echo "INSERT INTO system.numbers Select * from numbers(10);" \
+  | ${CLICKHOUSE_CURL} -sS "${CH_URL}&default_format=JSON" -i | grep 'xception\|Content-Type' | sed 's/Exception/Ex---tion/' | awk '{ print $1 $2 $3 }'
+echo ""
+echo "INSERT WITH default_format=XML"
+echo "INSERT INTO system.numbers Select * from numbers(10);" \
+  | ${CLICKHOUSE_CURL} -sS "${CH_URL}&default_format=XML" -i | grep 'xception\|Content-Type' | sed 's/Exception/Ex---tion/' | awk '{ print $1 $2 $3 }'
+echo ""
+echo "INSERT WITH default_format=BADFORMAT"
+echo "INSERT INTO system.numbers Select * from numbers(10);" \
+  | ${CLICKHOUSE_CURL} -sS "${CH_URL}&default_format=BADFORMAT" -i | grep 'xception\|Content-Type' | sed 's/Exception/Ex---tion/' | awk '{ print $1 $2 $3 }'
+
+
+echo ""
+echo "INSERT WITH X-ClickHouse-Format: JSON"
+echo "INSERT INTO system.numbers Select * from numbers(10);" \
+  | ${CLICKHOUSE_CURL} -sS "${CH_URL}" -H 'X-ClickHouse-Format: JSON' -i | grep 'xception\|Content-Type' | sed 's/Exception/Ex---tion/' | awk '{ print $1 $2 $3 }'
+echo ""
+echo "INSERT WITH X-ClickHouse-Format: XML"
+echo "INSERT INTO system.numbers Select * from numbers(10);" \
+  | ${CLICKHOUSE_CURL} -sS "${CH_URL}" -H 'X-ClickHouse-Format: XML' -i | grep 'xception\|Content-Type' | sed 's/Exception/Ex---tion/' | awk '{ print $1 $2 $3 }'
+echo ""
+echo "INSERT WITH X-ClickHouse-Format: BADFORMAT"
+echo "INSERT INTO system.numbers Select * from numbers(10);" \
+  | ${CLICKHOUSE_CURL} -sS "${CH_URL}" -H 'X-ClickHouse-Format: BADFORMAT' -i | grep 'xception\|Content-Type' | sed 's/Exception/Ex---tion/' | awk '{ print $1 $2 $3 }'