diff --git a/tests/performance/point_in_polygon.xml b/tests/performance/point_in_polygon.xml
index 4f811530cbe..31c24eb006f 100644
--- a/tests/performance/point_in_polygon.xml
+++ b/tests/performance/point_in_polygon.xml
@@ -1,11 +1,19 @@
 <test>
+    <settings>
+        <!--
+            Not sure why it's needed. Maybe it has something to do with the
+            resulting block size of Memory table.
+         -->
+        <optimize_trivial_insert_select>0</optimize_trivial_insert_select>
+    </settings>
+
     <create_query>CREATE TABLE polygons (polygon Array(Array(Tuple(Float64, Float64)))) ENGINE = Memory</create_query>
     <create_query>
         INSERT INTO polygons
         WITH number + 1 AS radius
         SELECT [arrayMap(x -> (cos(x / 90. * pi()) * radius, sin(x / 90. * pi()) * radius), range(180))]
         FROM numbers_mt(5000000)
-        SETTINGS max_insert_threads = 4, max_memory_usage = 20000000000
+        SETTINGS max_insert_threads = 2, max_memory_usage = 30000000000
     </create_query>
 
     <query>SELECT pointInPolygon((100, 100), polygon) FROM polygons FORMAT Null</query>