ClickHouse/tests/performance/norm_distance.xml

<test>

    <substitutions>
        <substitution>
            <name>element_type</name>
            <values>
                <!-- integer cases are not relevant in practice for vector search, disable to reduce test runtime -->
                <!-- <value>UInt8</value> -->
                <!-- <value>Int16</value> -->
                <!-- <value>Int32</value> -->
                <!-- <value>Int64</value> -->
                <value>Float32</value>
                <value>Float64</value>
            </values>
        </substitution>
    </substitutions>

    <create_query>
        CREATE TABLE vecs_{element_type} (
            v Array({element_type})
        ) ENGINE=Memory;
    </create_query>

    <!-- Gererate arrays with random data -->
    <!-- Dimension = 150 is realistic for vector search use cases -->

    <fill_query>
        INSERT INTO vecs_{element_type}
        SELECT v FROM (
            SELECT
                number AS n,
                [
                    rand(n*10), rand(n*10+1), rand(n*10+2), rand(n*10+3), rand(n*10+4), rand(n*10+5), rand(n*10+6), rand(n*10+7), rand(n*10+8), rand(n*10+9),
                    rand(n*10+10), rand(n*10+11), rand(n*10+12), rand(n*10+13), rand(n*10+14), rand(n*10+15), rand(n*10+16), rand(n*10+17), rand(n*10+18), rand(n*10+19),
                    rand(n*10+20), rand(n*10+21), rand(n*10+22), rand(n*10+23), rand(n*10+24), rand(n*10+25), rand(n*10+26), rand(n*10+27), rand(n*10+28), rand(n*10+29),
                    rand(n*10+30), rand(n*10+31), rand(n*10+32), rand(n*10+33), rand(n*10+34), rand(n*10+35), rand(n*10+36), rand(n*10+37), rand(n*10+38), rand(n*10+39),
                    rand(n*10+40), rand(n*10+41), rand(n*10+42), rand(n*10+43), rand(n*10+44), rand(n*10+45), rand(n*10+46), rand(n*10+47), rand(n*10+48), rand(n*10+49),
                    rand(n*10+50), rand(n*10+51), rand(n*10+52), rand(n*10+53), rand(n*10+54), rand(n*10+55), rand(n*10+56), rand(n*10+57), rand(n*10+58), rand(n*10+59),
                    rand(n*10+60), rand(n*10+61), rand(n*10+62), rand(n*10+63), rand(n*10+64), rand(n*10+65), rand(n*10+66), rand(n*10+67), rand(n*10+68), rand(n*10+69),
                    rand(n*10+70), rand(n*10+71), rand(n*10+72), rand(n*10+73), rand(n*10+74), rand(n*10+75), rand(n*10+76), rand(n*10+77), rand(n*10+78), rand(n*10+79),
                    rand(n*10+80), rand(n*10+81), rand(n*10+82), rand(n*10+83), rand(n*10+84), rand(n*10+85), rand(n*10+86), rand(n*10+87), rand(n*10+88), rand(n*10+89),
                    rand(n*10+90), rand(n*10+91), rand(n*10+92), rand(n*10+93), rand(n*10+94), rand(n*10+95), rand(n*10+96), rand(n*10+97), rand(n*10+98), rand(n*10+99),
                    rand(n*10+100), rand(n*10+101), rand(n*10+102), rand(n*10+103), rand(n*10+104), rand(n*10+105), rand(n*10+106), rand(n*10+107), rand(n*10+108), rand(n*10+109),
                    rand(n*10+110), rand(n*10+111), rand(n*10+112), rand(n*10+113), rand(n*10+114), rand(n*10+115), rand(n*10+116), rand(n*10+117), rand(n*10+118), rand(n*10+119),
                    rand(n*10+120), rand(n*10+121), rand(n*10+122), rand(n*10+123), rand(n*10+124), rand(n*10+125), rand(n*10+126), rand(n*10+127), rand(n*10+128), rand(n*10+129),
                    rand(n*10+130), rand(n*10+131), rand(n*10+132), rand(n*10+133), rand(n*10+134), rand(n*10+135), rand(n*10+136), rand(n*10+137), rand(n*10+138), rand(n*10+139),
                    rand(n*10+140), rand(n*10+141), rand(n*10+142), rand(n*10+143), rand(n*10+144), rand(n*10+145), rand(n*10+146), rand(n*10+147), rand(n*10+148), rand(n*10+149)
                ] AS v
            FROM system.numbers
            LIMIT 5000000
        );
    </fill_query>

    <settings>
        <max_threads>1</max_threads>
    </settings>

    <!-- Norm kinds-->
    <substitutions>
        <substitution>
            <name>norm</name>
            <values>
                <value>L1</value>
                <value>L2</value>
                <value>Linf</value>
            </values>
        </substitution>
    </substitutions>

    <!-- Arrays -->
    <query>SELECT sum(dist) FROM (SELECT {norm}Norm(v) AS dist FROM vecs_{element_type})</query>
    <query>WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT {norm}Distance(a, v) AS dist FROM vecs_{element_type})</query>
    <query>WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT cosineDistance(a, v) AS dist FROM vecs_{element_type})</query>

    <drop_query>DROP TABLE vecs_{element_type}</drop_query>

</test>
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00			`<test>`
Test with different element types 2022-05-23 12:08:15 +00:00
			`<substitutions>`
			`<substitution>`
			`<name>element_type</name>`
			`<values>`
Add performance test for dotProduct() 2024-02-20 21:41:10 +00:00			`<!-- integer cases are not relevant in practice for vector search, disable to reduce test runtime -->`
Further reduce runtime of norm_distance.xml 2024-01-23 10:52:52 +00:00			`<!-- <value>UInt8</value> -->`
			`<!-- <value>Int16</value> -->`
Add performance test for dotProduct() 2024-02-20 21:41:10 +00:00			`<!-- <value>Int32</value> -->`
			`<!-- <value>Int64</value> -->`
Test with different element types 2022-05-23 12:08:15 +00:00			`<value>Float32</value>`
			`<value>Float64</value>`
			`</values>`
			`</substitution>`
			`</substitutions>`

Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00			`<create_query>`
Test with different element types 2022-05-23 12:08:15 +00:00			`CREATE TABLE vecs_{element_type} (`
			`v Array({element_type})`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00			`) ENGINE=Memory;`
			`</create_query>`

			`<!-- Gererate arrays with random data -->`
Reduce memory consumption of norm_distance.xml The test data was stored in in-memory Array and Tuple tables, each consuming about 80 GB. The population of the Tuple tables exceeded the maximum memory thresholds (*). As a fix, - Reduce the Array cardinality from 200 to 150 and the number of rows from 10 to 8 million, reducing memory consumption to ca. 50 GB. This should pass the memory threshold and does not affect the test purpose. - Don't test tuples. Due to their data layout, vector search is not efficient anyways. Saves another 80 GB. 2024-01-19 12:52:01 +00:00			`<!-- Dimension = 150 is realistic for vector search use cases -->`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00
Test with different element types 2022-05-23 12:08:15 +00:00			`<fill_query>`
			`INSERT INTO vecs_{element_type}`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00			`SELECT v FROM (`
			`SELECT`
			`number AS n,`
			`[`
Extend performance test norm_dist.xml Dimension = 200 (instead of 10) is more realistic for vector search use cases. 2024-01-17 18:15:26 +00:00			`rand(n10), rand(n10+1), rand(n10+2), rand(n10+3), rand(n10+4), rand(n10+5), rand(n10+6), rand(n10+7), rand(n10+8), rand(n10+9),`
			`rand(n10+10), rand(n10+11), rand(n10+12), rand(n10+13), rand(n10+14), rand(n10+15), rand(n10+16), rand(n10+17), rand(n10+18), rand(n10+19),`
			`rand(n10+20), rand(n10+21), rand(n10+22), rand(n10+23), rand(n10+24), rand(n10+25), rand(n10+26), rand(n10+27), rand(n10+28), rand(n10+29),`
			`rand(n10+30), rand(n10+31), rand(n10+32), rand(n10+33), rand(n10+34), rand(n10+35), rand(n10+36), rand(n10+37), rand(n10+38), rand(n10+39),`
			`rand(n10+40), rand(n10+41), rand(n10+42), rand(n10+43), rand(n10+44), rand(n10+45), rand(n10+46), rand(n10+47), rand(n10+48), rand(n10+49),`
			`rand(n10+50), rand(n10+51), rand(n10+52), rand(n10+53), rand(n10+54), rand(n10+55), rand(n10+56), rand(n10+57), rand(n10+58), rand(n10+59),`
			`rand(n10+60), rand(n10+61), rand(n10+62), rand(n10+63), rand(n10+64), rand(n10+65), rand(n10+66), rand(n10+67), rand(n10+68), rand(n10+69),`
			`rand(n10+70), rand(n10+71), rand(n10+72), rand(n10+73), rand(n10+74), rand(n10+75), rand(n10+76), rand(n10+77), rand(n10+78), rand(n10+79),`
			`rand(n10+80), rand(n10+81), rand(n10+82), rand(n10+83), rand(n10+84), rand(n10+85), rand(n10+86), rand(n10+87), rand(n10+88), rand(n10+89),`
			`rand(n10+90), rand(n10+91), rand(n10+92), rand(n10+93), rand(n10+94), rand(n10+95), rand(n10+96), rand(n10+97), rand(n10+98), rand(n10+99),`
			`rand(n10+100), rand(n10+101), rand(n10+102), rand(n10+103), rand(n10+104), rand(n10+105), rand(n10+106), rand(n10+107), rand(n10+108), rand(n10+109),`
			`rand(n10+110), rand(n10+111), rand(n10+112), rand(n10+113), rand(n10+114), rand(n10+115), rand(n10+116), rand(n10+117), rand(n10+118), rand(n10+119),`
			`rand(n10+120), rand(n10+121), rand(n10+122), rand(n10+123), rand(n10+124), rand(n10+125), rand(n10+126), rand(n10+127), rand(n10+128), rand(n10+129),`
			`rand(n10+130), rand(n10+131), rand(n10+132), rand(n10+133), rand(n10+134), rand(n10+135), rand(n10+136), rand(n10+137), rand(n10+138), rand(n10+139),`
Reduce memory consumption of norm_distance.xml The test data was stored in in-memory Array and Tuple tables, each consuming about 80 GB. The population of the Tuple tables exceeded the maximum memory thresholds (*). As a fix, - Reduce the Array cardinality from 200 to 150 and the number of rows from 10 to 8 million, reducing memory consumption to ca. 50 GB. This should pass the memory threshold and does not affect the test purpose. - Don't test tuples. Due to their data layout, vector search is not efficient anyways. Saves another 80 GB. 2024-01-19 12:52:01 +00:00			`rand(n10+140), rand(n10+141), rand(n10+142), rand(n10+143), rand(n10+144), rand(n10+145), rand(n10+146), rand(n10+147), rand(n10+148), rand(n10+149)`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00			`] AS v`
			`FROM system.numbers`
Further reduce runtime of norm_distance.xml 2024-01-23 10:52:52 +00:00			`LIMIT 5000000`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00			`);`
Test with different element types 2022-05-23 12:08:15 +00:00			`</fill_query>`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00
Test with max_threads=1 2022-05-23 15:01:19 +00:00			`<settings>`
			`<max_threads>1</max_threads>`
			`</settings>`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00
Parameterized with norm kind 2022-05-23 16:27:41 +00:00			`<!-- Norm kinds-->`
			`<substitutions>`
			`<substitution>`
			`<name>norm</name>`
			`<values>`
			`<value>L1</value>`
			`<value>L2</value>`
			`<value>Linf</value>`
			`</values>`
			`</substitution>`
			`</substitutions>`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00
			`<!-- Arrays -->`
Use same norm and distance function names for tuples and arrays 2022-05-25 16:09:03 +00:00			`<query>SELECT sum(dist) FROM (SELECT {norm}Norm(v) AS dist FROM vecs_{element_type})</query>`
			`<query>WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT {norm}Distance(a, v) AS dist FROM vecs_{element_type})</query>`
			`<query>WITH (SELECT v FROM vecs_{element_type} limit 1) AS a SELECT sum(dist) FROM (SELECT cosineDistance(a, v) AS dist FROM vecs_{element_type})</query>`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00
Test with different element types 2022-05-23 12:08:15 +00:00			`<drop_query>DROP TABLE vecs_{element_type}</drop_query>`
Perf test for Norm and Distance fuctions for arrays and tuples 2022-05-23 08:18:24 +00:00
			`</test>`