From 25ceb1df658d0676069676a04a4a2bde15666d3a Mon Sep 17 00:00:00 2001
From: Pavel Kruglov <avogar@avogar-dev.sas.yp-c.yandex.net>
Date: Thu, 13 May 2021 16:50:15 +0300
Subject: [PATCH] Final fixes

---
 src/Columns/IColumn.cpp                          |   5 -----
 src/Columns/IColumn.h                            |   2 --
 .../Formats/Impl/ArrowColumnToCHColumn.cpp       |   2 +-
 .../Formats/Impl/CHColumnToArrowColumn.cpp       |   1 -
 .../Formats/Impl/ORCBlockInputFormat.cpp         |   2 +-
 .../0_stateless/00900_orc_arrays_load.reference  |   8 ++++----
 .../queries/0_stateless/00900_orc_arrays_load.sh |   6 +++---
 .../00900_orc_nested_arrays_load.reference       |   4 ++--
 .../0_stateless/00900_orc_nested_arrays_load.sh  |   4 ++--
 .../00900_orc_nullable_arrays_load.reference     |  12 ++++++------
 .../00900_orc_nullable_arrays_load.sh            |   4 ++--
 .../0_stateless/test_6pbz8e/nested_arrays.orc    | Bin 0 -> 1454 bytes
 .../0_stateless/test_6pbz8e/nullable_arrays.orc  | Bin 0 -> 821 bytes
 13 files changed, 21 insertions(+), 29 deletions(-)
 create mode 100644 tests/queries/0_stateless/test_6pbz8e/nested_arrays.orc
 create mode 100644 tests/queries/0_stateless/test_6pbz8e/nullable_arrays.orc

diff --git a/src/Columns/IColumn.cpp b/src/Columns/IColumn.cpp
index 2e61dc50fa4..2fe19d5d9fd 100644
--- a/src/Columns/IColumn.cpp
+++ b/src/Columns/IColumn.cpp
@@ -41,9 +41,4 @@ bool isColumnConst(const IColumn & column)
     return checkColumn<ColumnConst>(column);
 }
 
-bool isColumnArray(const IColumn & column)
-{
-    return checkColumn<ColumnArray>(column);
-}
-
 }
diff --git a/src/Columns/IColumn.h b/src/Columns/IColumn.h
index a788191e49f..23acc81e63d 100644
--- a/src/Columns/IColumn.h
+++ b/src/Columns/IColumn.h
@@ -533,6 +533,4 @@ bool isColumnConst(const IColumn & column);
 /// True if column's an ColumnNullable instance. It's just a syntax sugar for type check.
 bool isColumnNullable(const IColumn & column);
 
-bool isColumnArray(const IColumn & column);
-
 }
diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
index 3134fe7d919..f5a4ed89387 100644
--- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
+++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp
@@ -272,7 +272,7 @@ namespace DB
         }
 
         // TODO: check if a column is const?
-        if (!is_nullable && !isColumnArray(internal_column) && arrow_column->null_count())
+        if (!is_nullable && !checkColumn<ColumnArray>(internal_column) && arrow_column->null_count())
         {
             throw Exception
                 {
diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp
index 3d0b16e3b0a..9772f6c5c2f 100644
--- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp
+++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp
@@ -18,7 +18,6 @@
 #include <arrow/util/decimal.h>
 #include <DataTypes/DataTypeLowCardinality.h>
 
-#include <common/logger_useful.h>
 
 namespace DB
 {
diff --git a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp
index 6f43addc4ed..07edaa551fb 100644
--- a/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp
+++ b/src/Processors/Formats/Impl/ORCBlockInputFormat.cpp
@@ -80,7 +80,7 @@ void ORCBlockInputFormat::prepareReader()
     {
         if (getPort().getHeader().has(schema->field(i)->name()))
         {
-            include_indices.push_back(i+1);
+            include_indices.push_back(i);
         }
     }
 }
diff --git a/tests/queries/0_stateless/00900_orc_arrays_load.reference b/tests/queries/0_stateless/00900_orc_arrays_load.reference
index 35df4fa923c..9b20ef98164 100644
--- a/tests/queries/0_stateless/00900_orc_arrays_load.reference
+++ b/tests/queries/0_stateless/00900_orc_arrays_load.reference
@@ -1,4 +1,4 @@
-1	[1,-2,3]	[1,2,3]	[100,-200,300]	[100,200,300]	[10000000,-20000000,30000000]	[10000000,2000000,3000000]	[100000000000000,-200000000000,3000000000000]	[100000000000000,20000000000000,3000000000000]	['Some string','Some string','Some string']	['0000','1111','2222']	[42.42,424.2,0.4242]	[424242.424242,4242042420.242424,42]	['2000-01-01','2001-01-01','2002-01-01']	['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00']	[0.20,10.00,4.00]	[4.00,10000.10,10000.10]	[1000000000.00,90.00,101001.01]
-2	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]
-1	[1,-2,3]	[1,2,3]	[100,-200,300]	[100,200,300]	[10000000,-20000000,30000000]	[10000000,2000000,3000000]	[100000000000000,-200000000000,3000000000000]	[100000000000000,20000000000000,3000000000000]	['Some string','Some string','Some string']	['0000','1111','2222']	[42.42,424.2,0.4242]	[424242.424242,4242042420.242424,42]	['2000-01-01','2001-01-01','2002-01-01']	['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00']	[0.20,10.00,4.00]	[4.00,10000.10,10000.10]	[1000000000.00,90.00,101001.01]
-2	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]
+[1,-2,3]	[1,2,3]	[100,-200,300]	[100,200,300]	[10000000,-20000000,30000000]	[10000000,2000000,3000000]	[100000000000000,-200000000000,3000000000000]	[100000000000000,20000000000000,3000000000000]	['Some string','Some string','Some string']	['0000','1111','2222']	[42.42,424.2,0.4242]	[424242.424242,4242042420.242424,42]	['2000-01-01','2001-01-01','2002-01-01']	['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00']	[0.20,10.00,4.00]	[4.00,10000.10,10000.10]	[1000000000.00,90.00,101001.01]
+[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]
+[1,-2,3]	[1,2,3]	[100,-200,300]	[100,200,300]	[10000000,-20000000,30000000]	[10000000,2000000,3000000]	[100000000000000,-200000000000,3000000000000]	[100000000000000,20000000000000,3000000000000]	['Some string','Some string','Some string']	['0000','1111','2222']	[42.42,424.2,0.4242]	[424242.424242,4242042420.242424,42]	['2000-01-01','2001-01-01','2002-01-01']	['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00']	[0.20,10.00,4.00]	[4.00,10000.10,10000.10]	[1000000000.00,90.00,101001.01]
+[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]	[]
diff --git a/tests/queries/0_stateless/00900_orc_arrays_load.sh b/tests/queries/0_stateless/00900_orc_arrays_load.sh
index b1b0e897156..410945b31eb 100755
--- a/tests/queries/0_stateless/00900_orc_arrays_load.sh
+++ b/tests/queries/0_stateless/00900_orc_arrays_load.sh
@@ -4,10 +4,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 # shellcheck source=../shell_config.sh
 . "$CUR_DIR"/../shell_config.sh
 
-${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_load"
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_arrays (id UInt32, arr1 Array(Int8), arr2 Array(UInt8), arr3 Array(Int16), arr4 Array(UInt16), arr5 Array(Int32), arr6 Array(UInt32), arr7 Array(Int64), arr8 Array(UInt64), arr9 Array(String), arr10 Array(FixedString(4)), arr11 Array(Float32), arr12 Array(Float64), arr13 Array(Date), arr14 Array(Datetime), arr15 Array(Decimal(4, 2)), arr16 Array(Decimal(10, 2)), arr17 Array(Decimal(25, 2))) ENGINE=MergeTree() order by id"
+${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_arrays"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_arrays (arr1 Array(Int8), arr2 Array(UInt8), arr3 Array(Int16), arr4 Array(UInt16), arr5 Array(Int32), arr6 Array(UInt32), arr7 Array(Int64), arr8 Array(UInt64), arr9 Array(String), arr10 Array(FixedString(4)), arr11 Array(Float32), arr12 Array(Float64), arr13 Array(Date), arr14 Array(Datetime), arr15 Array(Decimal(4, 2)), arr16 Array(Decimal(10, 2)), arr17 Array(Decimal(25, 2))) ENGINE=Memory()"
 
-${CLICKHOUSE_CLIENT} --query="INSERT INTO orc_arrays VALUES (1, [1,-2,3],[1,2,3],[100,-200,300],[100,200,300],[10000000,-20000000,30000000],[10000000,2000000,3000000],[100000000000000,-200000000000,3000000000000],[100000000000000,20000000000000,3000000000000],['Some string','Some string','Some string'],['0000','1111','2222'],[42.42,424.2,0.4242],[424242.424242,4242042420.242424,42],['2000-01-01','2001-01-01','2002-01-01'],['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'],[0.20,10.00,4.00],[4.00,10000.10,10000.10],[1000000000.00,90.00,101001.01]),(2, [],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[])"
+${CLICKHOUSE_CLIENT} --query="INSERT INTO orc_arrays VALUES ([1,-2,3],[1,2,3],[100,-200,300],[100,200,300],[10000000,-20000000,30000000],[10000000,2000000,3000000],[100000000000000,-200000000000,3000000000000],[100000000000000,20000000000000,3000000000000],['Some string','Some string','Some string'],['0000','1111','2222'],[42.42,424.2,0.4242],[424242.424242,4242042420.242424,42],['2000-01-01','2001-01-01','2002-01-01'],['2000-01-01 00:00:00','2001-01-01 00:00:00','2002-01-01 00:00:00'],[0.20,10.00,4.00],[4.00,10000.10,10000.10],[1000000000.00,90.00,101001.01]),([],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],[])"
 
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM orc_arrays FORMAT ORC" > "${CLICKHOUSE_TMP}"/arrays.orc
 
diff --git a/tests/queries/0_stateless/00900_orc_nested_arrays_load.reference b/tests/queries/0_stateless/00900_orc_nested_arrays_load.reference
index 78cf96eb37c..dd9c9900684 100644
--- a/tests/queries/0_stateless/00900_orc_nested_arrays_load.reference
+++ b/tests/queries/0_stateless/00900_orc_nested_arrays_load.reference
@@ -1,2 +1,2 @@
-1	[[[1,2,3],[1,2,3]],[[1,2,3]],[[],[1,2,3]]]	[[['Some string','Some string'],[]],[['Some string']],[[]]]	[[NULL,1,2],[NULL],[1,2],[]]	[['Some string',NULL,'Some string'],[NULL],[]]
-1	[[[1,2,3],[1,2,3]],[[1,2,3]],[[],[1,2,3]]]	[[['Some string','Some string'],[]],[['Some string']],[[]]]	[[NULL,1,2],[NULL],[1,2],[]]	[['Some string',NULL,'Some string'],[NULL],[]]
+[[[1,2,3],[1,2,3]],[[1,2,3]],[[],[1,2,3]]]	[[['Some string','Some string'],[]],[['Some string']],[[]]]	[[NULL,1,2],[NULL],[1,2],[]]	[['Some string',NULL,'Some string'],[NULL],[]]
+[[[1,2,3],[1,2,3]],[[1,2,3]],[[],[1,2,3]]]	[[['Some string','Some string'],[]],[['Some string']],[[]]]	[[NULL,1,2],[NULL],[1,2],[]]	[['Some string',NULL,'Some string'],[NULL],[]]
diff --git a/tests/queries/0_stateless/00900_orc_nested_arrays_load.sh b/tests/queries/0_stateless/00900_orc_nested_arrays_load.sh
index 8875a4686dd..81d2aeed4bd 100755
--- a/tests/queries/0_stateless/00900_orc_nested_arrays_load.sh
+++ b/tests/queries/0_stateless/00900_orc_nested_arrays_load.sh
@@ -5,9 +5,9 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_nested_arrays"
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_nested_arrays (id UInt32, arr1 Array(Array(Array(UInt32))), arr2 Array(Array(Array(String))), arr3 Array(Array(Nullable(UInt32))), arr4 Array(Array(Nullable(String)))) engine=MergeTree() ORDER BY id"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_nested_arrays (arr1 Array(Array(Array(UInt32))), arr2 Array(Array(Array(String))), arr3 Array(Array(Nullable(UInt32))), arr4 Array(Array(Nullable(String)))) engine=Memory()"
 
-${CLICKHOUSE_CLIENT} --query="INSERT INTO orc_nested_arrays VALUES (1, [[[1,2,3],[1,2,3]],[[1,2,3]],[[],[1,2,3]]],[[['Some string','Some string'],[]],[['Some string']],[[]]],[[NULL,1,2],[NULL],[1,2],[]],[['Some string',NULL,'Some string'],[NULL],[]])"
+${CLICKHOUSE_CLIENT} --query="INSERT INTO orc_nested_arrays VALUES ([[[1,2,3],[1,2,3]],[[1,2,3]],[[],[1,2,3]]],[[['Some string','Some string'],[]],[['Some string']],[[]]],[[NULL,1,2],[NULL],[1,2],[]],[['Some string',NULL,'Some string'],[NULL],[]])"
 
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM orc_nested_arrays FORMAT ORC" > "${CLICKHOUSE_TMP}"/nested_arrays.orc
 
diff --git a/tests/queries/0_stateless/00900_orc_nullable_arrays_load.reference b/tests/queries/0_stateless/00900_orc_nullable_arrays_load.reference
index d548488a196..62e95652040 100644
--- a/tests/queries/0_stateless/00900_orc_nullable_arrays_load.reference
+++ b/tests/queries/0_stateless/00900_orc_nullable_arrays_load.reference
@@ -1,6 +1,6 @@
-1	[1,NULL,2]	[NULL,'Some string',NULL]	[0.00,NULL,42.42]
-2	[NULL]	[NULL]	[NULL]
-3	[]	[]	[]
-1	[1,NULL,2]	[NULL,'Some string',NULL]	[0.00,NULL,42.42]
-2	[NULL]	[NULL]	[NULL]
-3	[]	[]	[]
+[1,NULL,2]	[NULL,'Some string',NULL]	[0.00,NULL,42.42]
+[NULL]	[NULL]	[NULL]
+[]	[]	[]
+[1,NULL,2]	[NULL,'Some string',NULL]	[0.00,NULL,42.42]
+[NULL]	[NULL]	[NULL]
+[]	[]	[]
diff --git a/tests/queries/0_stateless/00900_orc_nullable_arrays_load.sh b/tests/queries/0_stateless/00900_orc_nullable_arrays_load.sh
index 59fb7874515..61829d0709b 100755
--- a/tests/queries/0_stateless/00900_orc_nullable_arrays_load.sh
+++ b/tests/queries/0_stateless/00900_orc_nullable_arrays_load.sh
@@ -5,8 +5,8 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
 . "$CUR_DIR"/../shell_config.sh
 
 ${CLICKHOUSE_CLIENT} --query="DROP TABLE IF EXISTS orc_nullable_arrays"
-${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_nullable_arrays (id UInt32, arr1 Array(Nullable(UInt32)), arr2 Array(Nullable(String)), arr3 Array(Nullable(Decimal(4, 2)))) ENGINE=MergeTree() order by id"
-${CLICKHOUSE_CLIENT} --query="INSERT INTO orc_nullable_arrays VALUES (1, [1,NULL,2],[NULL,'Some string',NULL],[0.00,NULL,42.42]), (2, [NULL],[NULL],[NULL]), (3, [],[],[])"
+${CLICKHOUSE_CLIENT} --query="CREATE TABLE orc_nullable_arrays (arr1 Array(Nullable(UInt32)), arr2 Array(Nullable(String)), arr3 Array(Nullable(Decimal(4, 2)))) ENGINE=Memory()"
+${CLICKHOUSE_CLIENT} --query="INSERT INTO orc_nullable_arrays VALUES ([1,NULL,2],[NULL,'Some string',NULL],[0.00,NULL,42.42]), ([NULL],[NULL],[NULL]), ([],[],[])"
 ${CLICKHOUSE_CLIENT} --query="SELECT * FROM orc_nullable_arrays FORMAT ORC" > "${CLICKHOUSE_TMP}"/nullable_arrays.orc
 
 cat "${CLICKHOUSE_TMP}"/nullable_arrays.orc | ${CLICKHOUSE_CLIENT} -q "INSERT INTO orc_nullable_arrays FORMAT ORC"
diff --git a/tests/queries/0_stateless/test_6pbz8e/nested_arrays.orc b/tests/queries/0_stateless/test_6pbz8e/nested_arrays.orc
new file mode 100644
index 0000000000000000000000000000000000000000..d538b7ded10449d4a2255037996767aedea57a4d
GIT binary patch
literal 1454
zcmcIjOODe(5Uu{W-D%sgW$|gU3Ry*hMfj|;LNg24z(fNVKmwtW_(*{RM9c+7NbEV<
zTmWenELemw)#Zxmj%Jo9PCvhT^{T8=*U|BzE1Uv=R$H+GchhA^kd{`o#>z%|iQ#G&
z?z1AQQZ-9!lvKm`MVb{lU0*|^+B!qu0j;&_K-z=Uu~#5CqA%CfRr_)!;x>Dx@a1tE
ziEZw$M7P18#J}p!|41Rb-vH_opY8&L9G)RTbQfC;Lw62Py!`ax?ez1P^~uL~SHGf7
zqd{S%?)_G3skxBC3O_E3YbrQnO#{9n6#h1$^fjSk)QlPHhI!6t8AtvG`5g0tv1DJx
ze9Tx=K45=|K4=7?V1#f*Fiuy6bj4hGO<XhA-Vo2cLKIErOX7z09-+yZ=j^x4t<NZb
z#C*hcM*c3v9mjiwreI#MuOu#-l6l2?kI;;nkGXHnyyiZ&zlB?ZbNGMfL_K}}%58+}
z_bgq)KBjHBxY~wID{Z`BsUCLX-|p<R5G`%IAUh>gwU(E_s^(ihsSwVj|1PFK)m+T(
zVYf_3<En`~c{9<k*Xsv#KBV&zosXw5hXu^AHNO?swIU%pR<f%Nd7NWoG8zm?xQKJg
zAtz}(j&tfEhwCiFl7_KN#?CPsJHF%CQIE#Hr{jQ^!`KHcM3b=(3XUD~dF)TlUOW?a
YD{<8}8$FXy%umI_eEp@=(edG(zpQ7Ik^lez

literal 0
HcmV?d00001

diff --git a/tests/queries/0_stateless/test_6pbz8e/nullable_arrays.orc b/tests/queries/0_stateless/test_6pbz8e/nullable_arrays.orc
new file mode 100644
index 0000000000000000000000000000000000000000..51dc8689d252bdb7c2c778002f5844089f1b6bdf
GIT binary patch
literal 821
zcma)4yH3L}6!rCW9Ls6JiV9ens;WYrsu0T3scHvUpjBZ)EJz^K0gwtGQ1JzP0z!Ne
zAJCPZwGGB`LLXAa;&acr$2Yz@=?{A{kVps#nZN;&JJK`6(2~HA3#zy-8M&OxYc6VC
zO!EP(x<bkAED{O<T`G3hE%m|@37S+Si<@yAdab0{N}>z~kWSMn0h&%{2qb-*Q|8J}
z8k|4g-&UWV$D@b4yiY#5bH*eWToGNi2zt*V)!P=)apt(ED^}-@`+DDSV(wFmU=b8(
zw47Q_71h7rAqrg|U7xu=bA8Y83_*E1u=lvrx%2sMj&J99%Am=`yyO38gvX~B@=|6&
zEvJ}ki<xEGUqBh(kcL3@iIEws-a#K#XVAb}U6euwl<|m;ZX$L)9v|2<vFBlxG)PQp
zLZGVxAw$T?itH6`vFP>-dCOERrZKkdTiZ)J7TUs&PAhhFe8Y~N*RvB4UtoTK$%NG?
P2^v>1fB8o24}1Cts>@YL

literal 0
HcmV?d00001