From d57a8b502d7be2917087f844ca02c8e3d0de4c90 Mon Sep 17 00:00:00 2001 From: Suzy Wang Date: Fri, 25 Nov 2022 13:12:11 -0800 Subject: [PATCH 01/88] s390x crc32 fix --- src/Common/HashTable/Hash.h | 593 +++++++++++++++++++- src/Functions/FunctionsStringHash.cpp | 8 + src/Functions/FunctionsStringSimilarity.cpp | 2 + 3 files changed, 601 insertions(+), 2 deletions(-) diff --git a/src/Common/HashTable/Hash.h b/src/Common/HashTable/Hash.h index 189db68f59a..758286638ca 100644 --- a/src/Common/HashTable/Hash.h +++ b/src/Common/HashTable/Hash.h @@ -48,6 +48,563 @@ inline DB::UInt64 intHash64(DB::UInt64 x) #include #endif +#if defined(__s390x__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ +static const unsigned int __attribute__((aligned(128))) crc32table_be[8][256] = { + { + 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, + 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, + 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, + 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, + 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, + 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, + 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, + 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, + 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, + 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, + 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, + 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, + 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, + 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, + 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, + 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, + 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, + 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, + 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, + 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, + 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, + 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, + 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, + 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, + 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, + 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, + 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, + 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, + 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, + 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, + 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, + 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, + 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, + 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, + 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, + 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, + 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, + 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, + 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, + 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, + 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, + 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, + 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, + 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, + 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, + 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, + 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, + 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, + 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, + 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, + 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, + 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, + 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, + 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, + 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, + 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, + 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, + 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, + 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, + 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, + 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, + 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, + 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, + 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 + },{ + 0x00000000, 0xd219c1dc, 0xa0f29e0f, 0x72eb5fd3, + 0x452421a9, 0x973de075, 0xe5d6bfa6, 0x37cf7e7a, + 0x8a484352, 0x5851828e, 0x2abadd5d, 0xf8a31c81, + 0xcf6c62fb, 0x1d75a327, 0x6f9efcf4, 0xbd873d28, + 0x10519b13, 0xc2485acf, 0xb0a3051c, 0x62bac4c0, + 0x5575baba, 0x876c7b66, 0xf58724b5, 0x279ee569, + 0x9a19d841, 0x4800199d, 0x3aeb464e, 0xe8f28792, + 0xdf3df9e8, 0x0d243834, 0x7fcf67e7, 0xadd6a63b, + 0x20a33626, 0xf2baf7fa, 0x8051a829, 0x524869f5, + 0x6587178f, 0xb79ed653, 0xc5758980, 0x176c485c, + 0xaaeb7574, 0x78f2b4a8, 0x0a19eb7b, 0xd8002aa7, + 0xefcf54dd, 0x3dd69501, 0x4f3dcad2, 0x9d240b0e, + 0x30f2ad35, 0xe2eb6ce9, 0x9000333a, 0x4219f2e6, + 0x75d68c9c, 0xa7cf4d40, 0xd5241293, 0x073dd34f, + 0xbabaee67, 0x68a32fbb, 0x1a487068, 0xc851b1b4, + 0xff9ecfce, 0x2d870e12, 0x5f6c51c1, 0x8d75901d, + 0x41466c4c, 0x935fad90, 0xe1b4f243, 0x33ad339f, + 0x04624de5, 0xd67b8c39, 0xa490d3ea, 0x76891236, + 0xcb0e2f1e, 0x1917eec2, 0x6bfcb111, 0xb9e570cd, + 0x8e2a0eb7, 0x5c33cf6b, 0x2ed890b8, 0xfcc15164, + 0x5117f75f, 0x830e3683, 0xf1e56950, 0x23fca88c, + 0x1433d6f6, 0xc62a172a, 0xb4c148f9, 0x66d88925, + 0xdb5fb40d, 0x094675d1, 0x7bad2a02, 0xa9b4ebde, + 0x9e7b95a4, 0x4c625478, 0x3e890bab, 0xec90ca77, + 0x61e55a6a, 0xb3fc9bb6, 0xc117c465, 0x130e05b9, + 0x24c17bc3, 0xf6d8ba1f, 0x8433e5cc, 0x562a2410, + 0xebad1938, 0x39b4d8e4, 0x4b5f8737, 0x994646eb, + 0xae893891, 0x7c90f94d, 0x0e7ba69e, 0xdc626742, + 0x71b4c179, 0xa3ad00a5, 0xd1465f76, 0x035f9eaa, + 0x3490e0d0, 0xe689210c, 0x94627edf, 0x467bbf03, + 0xfbfc822b, 0x29e543f7, 0x5b0e1c24, 0x8917ddf8, + 0xbed8a382, 0x6cc1625e, 0x1e2a3d8d, 0xcc33fc51, + 0x828cd898, 0x50951944, 0x227e4697, 0xf067874b, + 0xc7a8f931, 0x15b138ed, 0x675a673e, 0xb543a6e2, + 0x08c49bca, 0xdadd5a16, 0xa83605c5, 0x7a2fc419, + 0x4de0ba63, 0x9ff97bbf, 0xed12246c, 0x3f0be5b0, + 0x92dd438b, 0x40c48257, 0x322fdd84, 0xe0361c58, + 0xd7f96222, 0x05e0a3fe, 0x770bfc2d, 0xa5123df1, + 0x189500d9, 0xca8cc105, 0xb8679ed6, 0x6a7e5f0a, + 0x5db12170, 0x8fa8e0ac, 0xfd43bf7f, 0x2f5a7ea3, + 0xa22feebe, 0x70362f62, 0x02dd70b1, 0xd0c4b16d, + 0xe70bcf17, 0x35120ecb, 0x47f95118, 0x95e090c4, + 0x2867adec, 0xfa7e6c30, 0x889533e3, 0x5a8cf23f, + 0x6d438c45, 0xbf5a4d99, 0xcdb1124a, 0x1fa8d396, + 0xb27e75ad, 0x6067b471, 0x128ceba2, 0xc0952a7e, + 0xf75a5404, 0x254395d8, 0x57a8ca0b, 0x85b10bd7, + 0x383636ff, 0xea2ff723, 0x98c4a8f0, 0x4add692c, + 0x7d121756, 0xaf0bd68a, 0xdde08959, 0x0ff94885, + 0xc3cab4d4, 0x11d37508, 0x63382adb, 0xb121eb07, + 0x86ee957d, 0x54f754a1, 0x261c0b72, 0xf405caae, + 0x4982f786, 0x9b9b365a, 0xe9706989, 0x3b69a855, + 0x0ca6d62f, 0xdebf17f3, 0xac544820, 0x7e4d89fc, + 0xd39b2fc7, 0x0182ee1b, 0x7369b1c8, 0xa1707014, + 0x96bf0e6e, 0x44a6cfb2, 0x364d9061, 0xe45451bd, + 0x59d36c95, 0x8bcaad49, 0xf921f29a, 0x2b383346, + 0x1cf74d3c, 0xceee8ce0, 0xbc05d333, 0x6e1c12ef, + 0xe36982f2, 0x3170432e, 0x439b1cfd, 0x9182dd21, + 0xa64da35b, 0x74546287, 0x06bf3d54, 0xd4a6fc88, + 0x6921c1a0, 0xbb38007c, 0xc9d35faf, 0x1bca9e73, + 0x2c05e009, 0xfe1c21d5, 0x8cf77e06, 0x5eeebfda, + 0xf33819e1, 0x2121d83d, 0x53ca87ee, 0x81d34632, + 0xb61c3848, 0x6405f994, 0x16eea647, 0xc4f7679b, + 0x79705ab3, 0xab699b6f, 0xd982c4bc, 0x0b9b0560, + 0x3c547b1a, 0xee4dbac6, 0x9ca6e515, 0x4ebf24c9 + },{ + 0x00000000, 0x01d8ac87, 0x03b1590e, 0x0269f589, + 0x0762b21c, 0x06ba1e9b, 0x04d3eb12, 0x050b4795, + 0x0ec56438, 0x0f1dc8bf, 0x0d743d36, 0x0cac91b1, + 0x09a7d624, 0x087f7aa3, 0x0a168f2a, 0x0bce23ad, + 0x1d8ac870, 0x1c5264f7, 0x1e3b917e, 0x1fe33df9, + 0x1ae87a6c, 0x1b30d6eb, 0x19592362, 0x18818fe5, + 0x134fac48, 0x129700cf, 0x10fef546, 0x112659c1, + 0x142d1e54, 0x15f5b2d3, 0x179c475a, 0x1644ebdd, + 0x3b1590e0, 0x3acd3c67, 0x38a4c9ee, 0x397c6569, + 0x3c7722fc, 0x3daf8e7b, 0x3fc67bf2, 0x3e1ed775, + 0x35d0f4d8, 0x3408585f, 0x3661add6, 0x37b90151, + 0x32b246c4, 0x336aea43, 0x31031fca, 0x30dbb34d, + 0x269f5890, 0x2747f417, 0x252e019e, 0x24f6ad19, + 0x21fdea8c, 0x2025460b, 0x224cb382, 0x23941f05, + 0x285a3ca8, 0x2982902f, 0x2beb65a6, 0x2a33c921, + 0x2f388eb4, 0x2ee02233, 0x2c89d7ba, 0x2d517b3d, + 0x762b21c0, 0x77f38d47, 0x759a78ce, 0x7442d449, + 0x714993dc, 0x70913f5b, 0x72f8cad2, 0x73206655, + 0x78ee45f8, 0x7936e97f, 0x7b5f1cf6, 0x7a87b071, + 0x7f8cf7e4, 0x7e545b63, 0x7c3daeea, 0x7de5026d, + 0x6ba1e9b0, 0x6a794537, 0x6810b0be, 0x69c81c39, + 0x6cc35bac, 0x6d1bf72b, 0x6f7202a2, 0x6eaaae25, + 0x65648d88, 0x64bc210f, 0x66d5d486, 0x670d7801, + 0x62063f94, 0x63de9313, 0x61b7669a, 0x606fca1d, + 0x4d3eb120, 0x4ce61da7, 0x4e8fe82e, 0x4f5744a9, + 0x4a5c033c, 0x4b84afbb, 0x49ed5a32, 0x4835f6b5, + 0x43fbd518, 0x4223799f, 0x404a8c16, 0x41922091, + 0x44996704, 0x4541cb83, 0x47283e0a, 0x46f0928d, + 0x50b47950, 0x516cd5d7, 0x5305205e, 0x52dd8cd9, + 0x57d6cb4c, 0x560e67cb, 0x54679242, 0x55bf3ec5, + 0x5e711d68, 0x5fa9b1ef, 0x5dc04466, 0x5c18e8e1, + 0x5913af74, 0x58cb03f3, 0x5aa2f67a, 0x5b7a5afd, + 0xec564380, 0xed8eef07, 0xefe71a8e, 0xee3fb609, + 0xeb34f19c, 0xeaec5d1b, 0xe885a892, 0xe95d0415, + 0xe29327b8, 0xe34b8b3f, 0xe1227eb6, 0xe0fad231, + 0xe5f195a4, 0xe4293923, 0xe640ccaa, 0xe798602d, + 0xf1dc8bf0, 0xf0042777, 0xf26dd2fe, 0xf3b57e79, + 0xf6be39ec, 0xf766956b, 0xf50f60e2, 0xf4d7cc65, + 0xff19efc8, 0xfec1434f, 0xfca8b6c6, 0xfd701a41, + 0xf87b5dd4, 0xf9a3f153, 0xfbca04da, 0xfa12a85d, + 0xd743d360, 0xd69b7fe7, 0xd4f28a6e, 0xd52a26e9, + 0xd021617c, 0xd1f9cdfb, 0xd3903872, 0xd24894f5, + 0xd986b758, 0xd85e1bdf, 0xda37ee56, 0xdbef42d1, + 0xdee40544, 0xdf3ca9c3, 0xdd555c4a, 0xdc8df0cd, + 0xcac91b10, 0xcb11b797, 0xc978421e, 0xc8a0ee99, + 0xcdaba90c, 0xcc73058b, 0xce1af002, 0xcfc25c85, + 0xc40c7f28, 0xc5d4d3af, 0xc7bd2626, 0xc6658aa1, + 0xc36ecd34, 0xc2b661b3, 0xc0df943a, 0xc10738bd, + 0x9a7d6240, 0x9ba5cec7, 0x99cc3b4e, 0x981497c9, + 0x9d1fd05c, 0x9cc77cdb, 0x9eae8952, 0x9f7625d5, + 0x94b80678, 0x9560aaff, 0x97095f76, 0x96d1f3f1, + 0x93dab464, 0x920218e3, 0x906bed6a, 0x91b341ed, + 0x87f7aa30, 0x862f06b7, 0x8446f33e, 0x859e5fb9, + 0x8095182c, 0x814db4ab, 0x83244122, 0x82fceda5, + 0x8932ce08, 0x88ea628f, 0x8a839706, 0x8b5b3b81, + 0x8e507c14, 0x8f88d093, 0x8de1251a, 0x8c39899d, + 0xa168f2a0, 0xa0b05e27, 0xa2d9abae, 0xa3010729, + 0xa60a40bc, 0xa7d2ec3b, 0xa5bb19b2, 0xa463b535, + 0xafad9698, 0xae753a1f, 0xac1ccf96, 0xadc46311, + 0xa8cf2484, 0xa9178803, 0xab7e7d8a, 0xaaa6d10d, + 0xbce23ad0, 0xbd3a9657, 0xbf5363de, 0xbe8bcf59, + 0xbb8088cc, 0xba58244b, 0xb831d1c2, 0xb9e97d45, + 0xb2275ee8, 0xb3fff26f, 0xb19607e6, 0xb04eab61, + 0xb545ecf4, 0xb49d4073, 0xb6f4b5fa, 0xb72c197d + },{ + 0x00000000, 0xdc6d9ab7, 0xbc1a28d9, 0x6077b26e, + 0x7cf54c05, 0xa098d6b2, 0xc0ef64dc, 0x1c82fe6b, + 0xf9ea980a, 0x258702bd, 0x45f0b0d3, 0x999d2a64, + 0x851fd40f, 0x59724eb8, 0x3905fcd6, 0xe5686661, + 0xf7142da3, 0x2b79b714, 0x4b0e057a, 0x97639fcd, + 0x8be161a6, 0x578cfb11, 0x37fb497f, 0xeb96d3c8, + 0x0efeb5a9, 0xd2932f1e, 0xb2e49d70, 0x6e8907c7, + 0x720bf9ac, 0xae66631b, 0xce11d175, 0x127c4bc2, + 0xeae946f1, 0x3684dc46, 0x56f36e28, 0x8a9ef49f, + 0x961c0af4, 0x4a719043, 0x2a06222d, 0xf66bb89a, + 0x1303defb, 0xcf6e444c, 0xaf19f622, 0x73746c95, + 0x6ff692fe, 0xb39b0849, 0xd3ecba27, 0x0f812090, + 0x1dfd6b52, 0xc190f1e5, 0xa1e7438b, 0x7d8ad93c, + 0x61082757, 0xbd65bde0, 0xdd120f8e, 0x017f9539, + 0xe417f358, 0x387a69ef, 0x580ddb81, 0x84604136, + 0x98e2bf5d, 0x448f25ea, 0x24f89784, 0xf8950d33, + 0xd1139055, 0x0d7e0ae2, 0x6d09b88c, 0xb164223b, + 0xade6dc50, 0x718b46e7, 0x11fcf489, 0xcd916e3e, + 0x28f9085f, 0xf49492e8, 0x94e32086, 0x488eba31, + 0x540c445a, 0x8861deed, 0xe8166c83, 0x347bf634, + 0x2607bdf6, 0xfa6a2741, 0x9a1d952f, 0x46700f98, + 0x5af2f1f3, 0x869f6b44, 0xe6e8d92a, 0x3a85439d, + 0xdfed25fc, 0x0380bf4b, 0x63f70d25, 0xbf9a9792, + 0xa31869f9, 0x7f75f34e, 0x1f024120, 0xc36fdb97, + 0x3bfad6a4, 0xe7974c13, 0x87e0fe7d, 0x5b8d64ca, + 0x470f9aa1, 0x9b620016, 0xfb15b278, 0x277828cf, + 0xc2104eae, 0x1e7dd419, 0x7e0a6677, 0xa267fcc0, + 0xbee502ab, 0x6288981c, 0x02ff2a72, 0xde92b0c5, + 0xcceefb07, 0x108361b0, 0x70f4d3de, 0xac994969, + 0xb01bb702, 0x6c762db5, 0x0c019fdb, 0xd06c056c, + 0x3504630d, 0xe969f9ba, 0x891e4bd4, 0x5573d163, + 0x49f12f08, 0x959cb5bf, 0xf5eb07d1, 0x29869d66, + 0xa6e63d1d, 0x7a8ba7aa, 0x1afc15c4, 0xc6918f73, + 0xda137118, 0x067eebaf, 0x660959c1, 0xba64c376, + 0x5f0ca517, 0x83613fa0, 0xe3168dce, 0x3f7b1779, + 0x23f9e912, 0xff9473a5, 0x9fe3c1cb, 0x438e5b7c, + 0x51f210be, 0x8d9f8a09, 0xede83867, 0x3185a2d0, + 0x2d075cbb, 0xf16ac60c, 0x911d7462, 0x4d70eed5, + 0xa81888b4, 0x74751203, 0x1402a06d, 0xc86f3ada, + 0xd4edc4b1, 0x08805e06, 0x68f7ec68, 0xb49a76df, + 0x4c0f7bec, 0x9062e15b, 0xf0155335, 0x2c78c982, + 0x30fa37e9, 0xec97ad5e, 0x8ce01f30, 0x508d8587, + 0xb5e5e3e6, 0x69887951, 0x09ffcb3f, 0xd5925188, + 0xc910afe3, 0x157d3554, 0x750a873a, 0xa9671d8d, + 0xbb1b564f, 0x6776ccf8, 0x07017e96, 0xdb6ce421, + 0xc7ee1a4a, 0x1b8380fd, 0x7bf43293, 0xa799a824, + 0x42f1ce45, 0x9e9c54f2, 0xfeebe69c, 0x22867c2b, + 0x3e048240, 0xe26918f7, 0x821eaa99, 0x5e73302e, + 0x77f5ad48, 0xab9837ff, 0xcbef8591, 0x17821f26, + 0x0b00e14d, 0xd76d7bfa, 0xb71ac994, 0x6b775323, + 0x8e1f3542, 0x5272aff5, 0x32051d9b, 0xee68872c, + 0xf2ea7947, 0x2e87e3f0, 0x4ef0519e, 0x929dcb29, + 0x80e180eb, 0x5c8c1a5c, 0x3cfba832, 0xe0963285, + 0xfc14ccee, 0x20795659, 0x400ee437, 0x9c637e80, + 0x790b18e1, 0xa5668256, 0xc5113038, 0x197caa8f, + 0x05fe54e4, 0xd993ce53, 0xb9e47c3d, 0x6589e68a, + 0x9d1cebb9, 0x4171710e, 0x2106c360, 0xfd6b59d7, + 0xe1e9a7bc, 0x3d843d0b, 0x5df38f65, 0x819e15d2, + 0x64f673b3, 0xb89be904, 0xd8ec5b6a, 0x0481c1dd, + 0x18033fb6, 0xc46ea501, 0xa419176f, 0x78748dd8, + 0x6a08c61a, 0xb6655cad, 0xd612eec3, 0x0a7f7474, + 0x16fd8a1f, 0xca9010a8, 0xaae7a2c6, 0x768a3871, + 0x93e25e10, 0x4f8fc4a7, 0x2ff876c9, 0xf395ec7e, + 0xef171215, 0x337a88a2, 0x530d3acc, 0x8f60a07b + },{ + 0x00000000, 0x490d678d, 0x921acf1a, 0xdb17a897, + 0x20f48383, 0x69f9e40e, 0xb2ee4c99, 0xfbe32b14, + 0x41e90706, 0x08e4608b, 0xd3f3c81c, 0x9afeaf91, + 0x611d8485, 0x2810e308, 0xf3074b9f, 0xba0a2c12, + 0x83d20e0c, 0xcadf6981, 0x11c8c116, 0x58c5a69b, + 0xa3268d8f, 0xea2bea02, 0x313c4295, 0x78312518, + 0xc23b090a, 0x8b366e87, 0x5021c610, 0x192ca19d, + 0xe2cf8a89, 0xabc2ed04, 0x70d54593, 0x39d8221e, + 0x036501af, 0x4a686622, 0x917fceb5, 0xd872a938, + 0x2391822c, 0x6a9ce5a1, 0xb18b4d36, 0xf8862abb, + 0x428c06a9, 0x0b816124, 0xd096c9b3, 0x999bae3e, + 0x6278852a, 0x2b75e2a7, 0xf0624a30, 0xb96f2dbd, + 0x80b70fa3, 0xc9ba682e, 0x12adc0b9, 0x5ba0a734, + 0xa0438c20, 0xe94eebad, 0x3259433a, 0x7b5424b7, + 0xc15e08a5, 0x88536f28, 0x5344c7bf, 0x1a49a032, + 0xe1aa8b26, 0xa8a7ecab, 0x73b0443c, 0x3abd23b1, + 0x06ca035e, 0x4fc764d3, 0x94d0cc44, 0xddddabc9, + 0x263e80dd, 0x6f33e750, 0xb4244fc7, 0xfd29284a, + 0x47230458, 0x0e2e63d5, 0xd539cb42, 0x9c34accf, + 0x67d787db, 0x2edae056, 0xf5cd48c1, 0xbcc02f4c, + 0x85180d52, 0xcc156adf, 0x1702c248, 0x5e0fa5c5, + 0xa5ec8ed1, 0xece1e95c, 0x37f641cb, 0x7efb2646, + 0xc4f10a54, 0x8dfc6dd9, 0x56ebc54e, 0x1fe6a2c3, + 0xe40589d7, 0xad08ee5a, 0x761f46cd, 0x3f122140, + 0x05af02f1, 0x4ca2657c, 0x97b5cdeb, 0xdeb8aa66, + 0x255b8172, 0x6c56e6ff, 0xb7414e68, 0xfe4c29e5, + 0x444605f7, 0x0d4b627a, 0xd65ccaed, 0x9f51ad60, + 0x64b28674, 0x2dbfe1f9, 0xf6a8496e, 0xbfa52ee3, + 0x867d0cfd, 0xcf706b70, 0x1467c3e7, 0x5d6aa46a, + 0xa6898f7e, 0xef84e8f3, 0x34934064, 0x7d9e27e9, + 0xc7940bfb, 0x8e996c76, 0x558ec4e1, 0x1c83a36c, + 0xe7608878, 0xae6deff5, 0x757a4762, 0x3c7720ef, + 0x0d9406bc, 0x44996131, 0x9f8ec9a6, 0xd683ae2b, + 0x2d60853f, 0x646de2b2, 0xbf7a4a25, 0xf6772da8, + 0x4c7d01ba, 0x05706637, 0xde67cea0, 0x976aa92d, + 0x6c898239, 0x2584e5b4, 0xfe934d23, 0xb79e2aae, + 0x8e4608b0, 0xc74b6f3d, 0x1c5cc7aa, 0x5551a027, + 0xaeb28b33, 0xe7bfecbe, 0x3ca84429, 0x75a523a4, + 0xcfaf0fb6, 0x86a2683b, 0x5db5c0ac, 0x14b8a721, + 0xef5b8c35, 0xa656ebb8, 0x7d41432f, 0x344c24a2, + 0x0ef10713, 0x47fc609e, 0x9cebc809, 0xd5e6af84, + 0x2e058490, 0x6708e31d, 0xbc1f4b8a, 0xf5122c07, + 0x4f180015, 0x06156798, 0xdd02cf0f, 0x940fa882, + 0x6fec8396, 0x26e1e41b, 0xfdf64c8c, 0xb4fb2b01, + 0x8d23091f, 0xc42e6e92, 0x1f39c605, 0x5634a188, + 0xadd78a9c, 0xe4daed11, 0x3fcd4586, 0x76c0220b, + 0xccca0e19, 0x85c76994, 0x5ed0c103, 0x17dda68e, + 0xec3e8d9a, 0xa533ea17, 0x7e244280, 0x3729250d, + 0x0b5e05e2, 0x4253626f, 0x9944caf8, 0xd049ad75, + 0x2baa8661, 0x62a7e1ec, 0xb9b0497b, 0xf0bd2ef6, + 0x4ab702e4, 0x03ba6569, 0xd8adcdfe, 0x91a0aa73, + 0x6a438167, 0x234ee6ea, 0xf8594e7d, 0xb15429f0, + 0x888c0bee, 0xc1816c63, 0x1a96c4f4, 0x539ba379, + 0xa878886d, 0xe175efe0, 0x3a624777, 0x736f20fa, + 0xc9650ce8, 0x80686b65, 0x5b7fc3f2, 0x1272a47f, + 0xe9918f6b, 0xa09ce8e6, 0x7b8b4071, 0x328627fc, + 0x083b044d, 0x413663c0, 0x9a21cb57, 0xd32cacda, + 0x28cf87ce, 0x61c2e043, 0xbad548d4, 0xf3d82f59, + 0x49d2034b, 0x00df64c6, 0xdbc8cc51, 0x92c5abdc, + 0x692680c8, 0x202be745, 0xfb3c4fd2, 0xb231285f, + 0x8be90a41, 0xc2e46dcc, 0x19f3c55b, 0x50fea2d6, + 0xab1d89c2, 0xe210ee4f, 0x390746d8, 0x700a2155, + 0xca000d47, 0x830d6aca, 0x581ac25d, 0x1117a5d0, + 0xeaf48ec4, 0xa3f9e949, 0x78ee41de, 0x31e32653 + },{ + 0x00000000, 0x1b280d78, 0x36501af0, 0x2d781788, + 0x6ca035e0, 0x77883898, 0x5af02f10, 0x41d82268, + 0xd9406bc0, 0xc26866b8, 0xef107130, 0xf4387c48, + 0xb5e05e20, 0xaec85358, 0x83b044d0, 0x989849a8, + 0xb641ca37, 0xad69c74f, 0x8011d0c7, 0x9b39ddbf, + 0xdae1ffd7, 0xc1c9f2af, 0xecb1e527, 0xf799e85f, + 0x6f01a1f7, 0x7429ac8f, 0x5951bb07, 0x4279b67f, + 0x03a19417, 0x1889996f, 0x35f18ee7, 0x2ed9839f, + 0x684289d9, 0x736a84a1, 0x5e129329, 0x453a9e51, + 0x04e2bc39, 0x1fcab141, 0x32b2a6c9, 0x299aabb1, + 0xb102e219, 0xaa2aef61, 0x8752f8e9, 0x9c7af591, + 0xdda2d7f9, 0xc68ada81, 0xebf2cd09, 0xf0dac071, + 0xde0343ee, 0xc52b4e96, 0xe853591e, 0xf37b5466, + 0xb2a3760e, 0xa98b7b76, 0x84f36cfe, 0x9fdb6186, + 0x0743282e, 0x1c6b2556, 0x311332de, 0x2a3b3fa6, + 0x6be31dce, 0x70cb10b6, 0x5db3073e, 0x469b0a46, + 0xd08513b2, 0xcbad1eca, 0xe6d50942, 0xfdfd043a, + 0xbc252652, 0xa70d2b2a, 0x8a753ca2, 0x915d31da, + 0x09c57872, 0x12ed750a, 0x3f956282, 0x24bd6ffa, + 0x65654d92, 0x7e4d40ea, 0x53355762, 0x481d5a1a, + 0x66c4d985, 0x7decd4fd, 0x5094c375, 0x4bbcce0d, + 0x0a64ec65, 0x114ce11d, 0x3c34f695, 0x271cfbed, + 0xbf84b245, 0xa4acbf3d, 0x89d4a8b5, 0x92fca5cd, + 0xd32487a5, 0xc80c8add, 0xe5749d55, 0xfe5c902d, + 0xb8c79a6b, 0xa3ef9713, 0x8e97809b, 0x95bf8de3, + 0xd467af8b, 0xcf4fa2f3, 0xe237b57b, 0xf91fb803, + 0x6187f1ab, 0x7aaffcd3, 0x57d7eb5b, 0x4cffe623, + 0x0d27c44b, 0x160fc933, 0x3b77debb, 0x205fd3c3, + 0x0e86505c, 0x15ae5d24, 0x38d64aac, 0x23fe47d4, + 0x622665bc, 0x790e68c4, 0x54767f4c, 0x4f5e7234, + 0xd7c63b9c, 0xccee36e4, 0xe196216c, 0xfabe2c14, + 0xbb660e7c, 0xa04e0304, 0x8d36148c, 0x961e19f4, + 0xa5cb3ad3, 0xbee337ab, 0x939b2023, 0x88b32d5b, + 0xc96b0f33, 0xd243024b, 0xff3b15c3, 0xe41318bb, + 0x7c8b5113, 0x67a35c6b, 0x4adb4be3, 0x51f3469b, + 0x102b64f3, 0x0b03698b, 0x267b7e03, 0x3d53737b, + 0x138af0e4, 0x08a2fd9c, 0x25daea14, 0x3ef2e76c, + 0x7f2ac504, 0x6402c87c, 0x497adff4, 0x5252d28c, + 0xcaca9b24, 0xd1e2965c, 0xfc9a81d4, 0xe7b28cac, + 0xa66aaec4, 0xbd42a3bc, 0x903ab434, 0x8b12b94c, + 0xcd89b30a, 0xd6a1be72, 0xfbd9a9fa, 0xe0f1a482, + 0xa12986ea, 0xba018b92, 0x97799c1a, 0x8c519162, + 0x14c9d8ca, 0x0fe1d5b2, 0x2299c23a, 0x39b1cf42, + 0x7869ed2a, 0x6341e052, 0x4e39f7da, 0x5511faa2, + 0x7bc8793d, 0x60e07445, 0x4d9863cd, 0x56b06eb5, + 0x17684cdd, 0x0c4041a5, 0x2138562d, 0x3a105b55, + 0xa28812fd, 0xb9a01f85, 0x94d8080d, 0x8ff00575, + 0xce28271d, 0xd5002a65, 0xf8783ded, 0xe3503095, + 0x754e2961, 0x6e662419, 0x431e3391, 0x58363ee9, + 0x19ee1c81, 0x02c611f9, 0x2fbe0671, 0x34960b09, + 0xac0e42a1, 0xb7264fd9, 0x9a5e5851, 0x81765529, + 0xc0ae7741, 0xdb867a39, 0xf6fe6db1, 0xedd660c9, + 0xc30fe356, 0xd827ee2e, 0xf55ff9a6, 0xee77f4de, + 0xafafd6b6, 0xb487dbce, 0x99ffcc46, 0x82d7c13e, + 0x1a4f8896, 0x016785ee, 0x2c1f9266, 0x37379f1e, + 0x76efbd76, 0x6dc7b00e, 0x40bfa786, 0x5b97aafe, + 0x1d0ca0b8, 0x0624adc0, 0x2b5cba48, 0x3074b730, + 0x71ac9558, 0x6a849820, 0x47fc8fa8, 0x5cd482d0, + 0xc44ccb78, 0xdf64c600, 0xf21cd188, 0xe934dcf0, + 0xa8ecfe98, 0xb3c4f3e0, 0x9ebce468, 0x8594e910, + 0xab4d6a8f, 0xb06567f7, 0x9d1d707f, 0x86357d07, + 0xc7ed5f6f, 0xdcc55217, 0xf1bd459f, 0xea9548e7, + 0x720d014f, 0x69250c37, 0x445d1bbf, 0x5f7516c7, + 0x1ead34af, 0x058539d7, 0x28fd2e5f, 0x33d52327 + },{ + 0x00000000, 0x4f576811, 0x9eaed022, 0xd1f9b833, + 0x399cbdf3, 0x76cbd5e2, 0xa7326dd1, 0xe86505c0, + 0x73397be6, 0x3c6e13f7, 0xed97abc4, 0xa2c0c3d5, + 0x4aa5c615, 0x05f2ae04, 0xd40b1637, 0x9b5c7e26, + 0xe672f7cc, 0xa9259fdd, 0x78dc27ee, 0x378b4fff, + 0xdfee4a3f, 0x90b9222e, 0x41409a1d, 0x0e17f20c, + 0x954b8c2a, 0xda1ce43b, 0x0be55c08, 0x44b23419, + 0xacd731d9, 0xe38059c8, 0x3279e1fb, 0x7d2e89ea, + 0xc824f22f, 0x87739a3e, 0x568a220d, 0x19dd4a1c, + 0xf1b84fdc, 0xbeef27cd, 0x6f169ffe, 0x2041f7ef, + 0xbb1d89c9, 0xf44ae1d8, 0x25b359eb, 0x6ae431fa, + 0x8281343a, 0xcdd65c2b, 0x1c2fe418, 0x53788c09, + 0x2e5605e3, 0x61016df2, 0xb0f8d5c1, 0xffafbdd0, + 0x17cab810, 0x589dd001, 0x89646832, 0xc6330023, + 0x5d6f7e05, 0x12381614, 0xc3c1ae27, 0x8c96c636, + 0x64f3c3f6, 0x2ba4abe7, 0xfa5d13d4, 0xb50a7bc5, + 0x9488f9e9, 0xdbdf91f8, 0x0a2629cb, 0x457141da, + 0xad14441a, 0xe2432c0b, 0x33ba9438, 0x7cedfc29, + 0xe7b1820f, 0xa8e6ea1e, 0x791f522d, 0x36483a3c, + 0xde2d3ffc, 0x917a57ed, 0x4083efde, 0x0fd487cf, + 0x72fa0e25, 0x3dad6634, 0xec54de07, 0xa303b616, + 0x4b66b3d6, 0x0431dbc7, 0xd5c863f4, 0x9a9f0be5, + 0x01c375c3, 0x4e941dd2, 0x9f6da5e1, 0xd03acdf0, + 0x385fc830, 0x7708a021, 0xa6f11812, 0xe9a67003, + 0x5cac0bc6, 0x13fb63d7, 0xc202dbe4, 0x8d55b3f5, + 0x6530b635, 0x2a67de24, 0xfb9e6617, 0xb4c90e06, + 0x2f957020, 0x60c21831, 0xb13ba002, 0xfe6cc813, + 0x1609cdd3, 0x595ea5c2, 0x88a71df1, 0xc7f075e0, + 0xbadefc0a, 0xf589941b, 0x24702c28, 0x6b274439, + 0x834241f9, 0xcc1529e8, 0x1dec91db, 0x52bbf9ca, + 0xc9e787ec, 0x86b0effd, 0x574957ce, 0x181e3fdf, + 0xf07b3a1f, 0xbf2c520e, 0x6ed5ea3d, 0x2182822c, + 0x2dd0ee65, 0x62878674, 0xb37e3e47, 0xfc295656, + 0x144c5396, 0x5b1b3b87, 0x8ae283b4, 0xc5b5eba5, + 0x5ee99583, 0x11befd92, 0xc04745a1, 0x8f102db0, + 0x67752870, 0x28224061, 0xf9dbf852, 0xb68c9043, + 0xcba219a9, 0x84f571b8, 0x550cc98b, 0x1a5ba19a, + 0xf23ea45a, 0xbd69cc4b, 0x6c907478, 0x23c71c69, + 0xb89b624f, 0xf7cc0a5e, 0x2635b26d, 0x6962da7c, + 0x8107dfbc, 0xce50b7ad, 0x1fa90f9e, 0x50fe678f, + 0xe5f41c4a, 0xaaa3745b, 0x7b5acc68, 0x340da479, + 0xdc68a1b9, 0x933fc9a8, 0x42c6719b, 0x0d91198a, + 0x96cd67ac, 0xd99a0fbd, 0x0863b78e, 0x4734df9f, + 0xaf51da5f, 0xe006b24e, 0x31ff0a7d, 0x7ea8626c, + 0x0386eb86, 0x4cd18397, 0x9d283ba4, 0xd27f53b5, + 0x3a1a5675, 0x754d3e64, 0xa4b48657, 0xebe3ee46, + 0x70bf9060, 0x3fe8f871, 0xee114042, 0xa1462853, + 0x49232d93, 0x06744582, 0xd78dfdb1, 0x98da95a0, + 0xb958178c, 0xf60f7f9d, 0x27f6c7ae, 0x68a1afbf, + 0x80c4aa7f, 0xcf93c26e, 0x1e6a7a5d, 0x513d124c, + 0xca616c6a, 0x8536047b, 0x54cfbc48, 0x1b98d459, + 0xf3fdd199, 0xbcaab988, 0x6d5301bb, 0x220469aa, + 0x5f2ae040, 0x107d8851, 0xc1843062, 0x8ed35873, + 0x66b65db3, 0x29e135a2, 0xf8188d91, 0xb74fe580, + 0x2c139ba6, 0x6344f3b7, 0xb2bd4b84, 0xfdea2395, + 0x158f2655, 0x5ad84e44, 0x8b21f677, 0xc4769e66, + 0x717ce5a3, 0x3e2b8db2, 0xefd23581, 0xa0855d90, + 0x48e05850, 0x07b73041, 0xd64e8872, 0x9919e063, + 0x02459e45, 0x4d12f654, 0x9ceb4e67, 0xd3bc2676, + 0x3bd923b6, 0x748e4ba7, 0xa577f394, 0xea209b85, + 0x970e126f, 0xd8597a7e, 0x09a0c24d, 0x46f7aa5c, + 0xae92af9c, 0xe1c5c78d, 0x303c7fbe, 0x7f6b17af, + 0xe4376989, 0xab600198, 0x7a99b9ab, 0x35ced1ba, + 0xddabd47a, 0x92fcbc6b, 0x43050458, 0x0c526c49 + },{ + 0x00000000, 0x5ba1dcca, 0xb743b994, 0xece2655e, + 0x6a466e9f, 0x31e7b255, 0xdd05d70b, 0x86a40bc1, + 0xd48cdd3e, 0x8f2d01f4, 0x63cf64aa, 0x386eb860, + 0xbecab3a1, 0xe56b6f6b, 0x09890a35, 0x5228d6ff, + 0xadd8a7cb, 0xf6797b01, 0x1a9b1e5f, 0x413ac295, + 0xc79ec954, 0x9c3f159e, 0x70dd70c0, 0x2b7cac0a, + 0x79547af5, 0x22f5a63f, 0xce17c361, 0x95b61fab, + 0x1312146a, 0x48b3c8a0, 0xa451adfe, 0xfff07134, + 0x5f705221, 0x04d18eeb, 0xe833ebb5, 0xb392377f, + 0x35363cbe, 0x6e97e074, 0x8275852a, 0xd9d459e0, + 0x8bfc8f1f, 0xd05d53d5, 0x3cbf368b, 0x671eea41, + 0xe1bae180, 0xba1b3d4a, 0x56f95814, 0x0d5884de, + 0xf2a8f5ea, 0xa9092920, 0x45eb4c7e, 0x1e4a90b4, + 0x98ee9b75, 0xc34f47bf, 0x2fad22e1, 0x740cfe2b, + 0x262428d4, 0x7d85f41e, 0x91679140, 0xcac64d8a, + 0x4c62464b, 0x17c39a81, 0xfb21ffdf, 0xa0802315, + 0xbee0a442, 0xe5417888, 0x09a31dd6, 0x5202c11c, + 0xd4a6cadd, 0x8f071617, 0x63e57349, 0x3844af83, + 0x6a6c797c, 0x31cda5b6, 0xdd2fc0e8, 0x868e1c22, + 0x002a17e3, 0x5b8bcb29, 0xb769ae77, 0xecc872bd, + 0x13380389, 0x4899df43, 0xa47bba1d, 0xffda66d7, + 0x797e6d16, 0x22dfb1dc, 0xce3dd482, 0x959c0848, + 0xc7b4deb7, 0x9c15027d, 0x70f76723, 0x2b56bbe9, + 0xadf2b028, 0xf6536ce2, 0x1ab109bc, 0x4110d576, + 0xe190f663, 0xba312aa9, 0x56d34ff7, 0x0d72933d, + 0x8bd698fc, 0xd0774436, 0x3c952168, 0x6734fda2, + 0x351c2b5d, 0x6ebdf797, 0x825f92c9, 0xd9fe4e03, + 0x5f5a45c2, 0x04fb9908, 0xe819fc56, 0xb3b8209c, + 0x4c4851a8, 0x17e98d62, 0xfb0be83c, 0xa0aa34f6, + 0x260e3f37, 0x7dafe3fd, 0x914d86a3, 0xcaec5a69, + 0x98c48c96, 0xc365505c, 0x2f873502, 0x7426e9c8, + 0xf282e209, 0xa9233ec3, 0x45c15b9d, 0x1e608757, + 0x79005533, 0x22a189f9, 0xce43eca7, 0x95e2306d, + 0x13463bac, 0x48e7e766, 0xa4058238, 0xffa45ef2, + 0xad8c880d, 0xf62d54c7, 0x1acf3199, 0x416eed53, + 0xc7cae692, 0x9c6b3a58, 0x70895f06, 0x2b2883cc, + 0xd4d8f2f8, 0x8f792e32, 0x639b4b6c, 0x383a97a6, + 0xbe9e9c67, 0xe53f40ad, 0x09dd25f3, 0x527cf939, + 0x00542fc6, 0x5bf5f30c, 0xb7179652, 0xecb64a98, + 0x6a124159, 0x31b39d93, 0xdd51f8cd, 0x86f02407, + 0x26700712, 0x7dd1dbd8, 0x9133be86, 0xca92624c, + 0x4c36698d, 0x1797b547, 0xfb75d019, 0xa0d40cd3, + 0xf2fcda2c, 0xa95d06e6, 0x45bf63b8, 0x1e1ebf72, + 0x98bab4b3, 0xc31b6879, 0x2ff90d27, 0x7458d1ed, + 0x8ba8a0d9, 0xd0097c13, 0x3ceb194d, 0x674ac587, + 0xe1eece46, 0xba4f128c, 0x56ad77d2, 0x0d0cab18, + 0x5f247de7, 0x0485a12d, 0xe867c473, 0xb3c618b9, + 0x35621378, 0x6ec3cfb2, 0x8221aaec, 0xd9807626, + 0xc7e0f171, 0x9c412dbb, 0x70a348e5, 0x2b02942f, + 0xada69fee, 0xf6074324, 0x1ae5267a, 0x4144fab0, + 0x136c2c4f, 0x48cdf085, 0xa42f95db, 0xff8e4911, + 0x792a42d0, 0x228b9e1a, 0xce69fb44, 0x95c8278e, + 0x6a3856ba, 0x31998a70, 0xdd7bef2e, 0x86da33e4, + 0x007e3825, 0x5bdfe4ef, 0xb73d81b1, 0xec9c5d7b, + 0xbeb48b84, 0xe515574e, 0x09f73210, 0x5256eeda, + 0xd4f2e51b, 0x8f5339d1, 0x63b15c8f, 0x38108045, + 0x9890a350, 0xc3317f9a, 0x2fd31ac4, 0x7472c60e, + 0xf2d6cdcf, 0xa9771105, 0x4595745b, 0x1e34a891, + 0x4c1c7e6e, 0x17bda2a4, 0xfb5fc7fa, 0xa0fe1b30, + 0x265a10f1, 0x7dfbcc3b, 0x9119a965, 0xcab875af, + 0x3548049b, 0x6ee9d851, 0x820bbd0f, 0xd9aa61c5, + 0x5f0e6a04, 0x04afb6ce, 0xe84dd390, 0xb3ec0f5a, + 0xe1c4d9a5, 0xba65056f, 0x56876031, 0x0d26bcfb, + 0x8b82b73a, 0xd0236bf0, 0x3cc10eae, 0x6760d264 + } +}; +inline unsigned int s390x_crc32_be(unsigned int crc, const unsigned char *buf, size_t len) { + while (len--) + crc = crc32table_be[0][((crc >> 24) ^ *buf++) & 0xFF] ^ (crc << 8); + return crc; +} + +inline uint32_t s390x_crc32_u8(uint32_t crc, uint8_t v) +{ + return s390x_crc32_be(crc, reinterpret_cast(&v), sizeof(v)); +} + +inline uint32_t s390x_crc32_u16(uint32_t crc, uint16_t v) +{ + return s390x_crc32_be(crc, reinterpret_cast(&v), sizeof(v)); +} + +inline uint32_t s390x_crc32_u32(uint32_t crc, uint32_t v) +{ + return s390x_crc32_be(crc, reinterpret_cast(&v), sizeof(v)); +} + +inline uint64_t s390x_crc32(uint64_t crc, uint64_t v) +{ + uint64_t _crc = crc; + uint32_t value_h, value_l; + value_h = (v >> 32) & 0xffffffff; + value_l = v & 0xffffffff; + _crc = s390x_crc32_be(static_cast(_crc), reinterpret_cast(&value_h), sizeof(uint32_t)); + _crc = s390x_crc32_be(static_cast(_crc), reinterpret_cast(&value_l), sizeof(uint32_t)); + return _crc; +} +#endif + /// NOTE: Intel intrinsic can be confusing. /// - https://code.google.com/archive/p/sse-intrinsics/wikis/PmovIntrinsicBug.wiki /// - https://stackoverflow.com/questions/15752770/mm-crc32-u64-poorly-defined @@ -57,6 +614,8 @@ inline DB::UInt64 intHashCRC32(DB::UInt64 x) return _mm_crc32_u64(-1ULL, x); #elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) return __crc32cd(-1U, x); +#elif defined(__s390x__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return s390x_crc32(-1U, x) #else /// On other platforms we do not have CRC32. NOTE This can be confusing. /// NOTE: consider using intHash32() @@ -69,6 +628,8 @@ inline DB::UInt64 intHashCRC32(DB::UInt64 x, DB::UInt64 updated_value) return _mm_crc32_u64(updated_value, x); #elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) return __crc32cd(static_cast(updated_value), x); +#elif defined(__s390x__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ + return s390x_crc32(updated_value, x); #else /// On other platforms we do not have CRC32. NOTE This can be confusing. return intHash64(x) ^ updated_value; @@ -101,25 +662,53 @@ inline UInt32 updateWeakHash32(const DB::UInt8 * pos, size_t size, DB::UInt32 up case 0: break; case 1: +#if __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ __builtin_memcpy(&value, pos, 1); +#else + reverseMemcpy(&value, pos, 1); +#endif break; case 2: +#if __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ __builtin_memcpy(&value, pos, 2); +#else + reverseMemcpy(&value, pos, 2); +#endif break; case 3: +#if __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ __builtin_memcpy(&value, pos, 3); +#else + reverseMemcpy(&value, pos, 3); +#endif break; case 4: +#if __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ __builtin_memcpy(&value, pos, 4); +#else + reverseMemcpy(&value, pos, 4); +#endif break; case 5: +#if __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ __builtin_memcpy(&value, pos, 5); +#else + reverseMemcpy(&value, pos, 5); +#endif break; case 6: +#if __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ __builtin_memcpy(&value, pos, 6); +#else + reverseMemcpy(&value, pos, 6); +#endif break; case 7: +#if __BYTE_ORDER__==__ORDER_LITTLE_ENDIAN__ __builtin_memcpy(&value, pos, 7); +#else + reverseMemcpy(&value, pos, 7); +#endif break; default: UNREACHABLE(); @@ -132,7 +721,7 @@ inline UInt32 updateWeakHash32(const DB::UInt8 * pos, size_t size, DB::UInt32 up const auto * end = pos + size; while (pos + 8 <= end) { - auto word = unalignedLoad(pos); + auto word = unalignedLoadLE(pos); updated_value = static_cast(intHashCRC32(word, updated_value)); pos += 8; @@ -144,7 +733,7 @@ inline UInt32 updateWeakHash32(const DB::UInt8 * pos, size_t size, DB::UInt32 up /// Lets' assume the string was 'abcdefghXYZ', so it's tail is 'XYZ'. DB::UInt8 tail_size = end - pos; /// Load tailing 8 bytes. Word is 'defghXYZ'. - auto word = unalignedLoad(end - 8); + auto word = unalignedLoadLE(end - 8); /// Prepare mask which will set other 5 bytes to 0. It is 0xFFFFFFFFFFFFFFFF << 5 = 0xFFFFFF0000000000. /// word & mask = '\0\0\0\0\0XYZ' (bytes are reversed because of little ending) word &= (~UInt64(0)) << DB::UInt8(8 * (8 - tail_size)); diff --git a/src/Functions/FunctionsStringHash.cpp b/src/Functions/FunctionsStringHash.cpp index 949503e2367..174acebe979 100644 --- a/src/Functions/FunctionsStringHash.cpp +++ b/src/Functions/FunctionsStringHash.cpp @@ -36,6 +36,8 @@ struct Hash return _mm_crc32_u64(crc, val); #elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) return __crc32cd(static_cast(crc), val); +#elif defined(__s390x__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ + return s390x_crc32(crc, val); #else throw Exception("String hash is not implemented without sse4.2 support", ErrorCodes::NOT_IMPLEMENTED); #endif @@ -47,6 +49,8 @@ struct Hash return _mm_crc32_u32(crc, val); #elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) return __crc32cw(crc, val); +#elif defined(__s390x__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ + return s390x_crc32_u32(crc, val); #else throw Exception("String hash is not implemented without sse4.2 support", ErrorCodes::NOT_IMPLEMENTED); #endif @@ -58,6 +62,8 @@ struct Hash return _mm_crc32_u16(crc, val); #elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) return __crc32ch(crc, val); +#elif defined(__s390x__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ + return s390x_crc32_u16(crc, val); #else throw Exception("String hash is not implemented without sse4.2 support", ErrorCodes::NOT_IMPLEMENTED); #endif @@ -69,6 +75,8 @@ struct Hash return _mm_crc32_u8(crc, val); #elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) return __crc32cb(crc, val); +#elif defined(__s390x__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ + return s390x_crc32_u8(crc, val); #else throw Exception("String hash is not implemented without sse4.2 support", ErrorCodes::NOT_IMPLEMENTED); #endif diff --git a/src/Functions/FunctionsStringSimilarity.cpp b/src/Functions/FunctionsStringSimilarity.cpp index d1bb77d6d40..802aafc2042 100644 --- a/src/Functions/FunctionsStringSimilarity.cpp +++ b/src/Functions/FunctionsStringSimilarity.cpp @@ -70,6 +70,8 @@ struct NgramDistanceImpl return _mm_crc32_u64(code_points[2], combined) & 0xFFFFu; #elif defined(__aarch64__) && defined(__ARM_FEATURE_CRC32) return __crc32cd(code_points[2], combined) & 0xFFFFu; +#elif defined(__s390x__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ + return s390x_crc32(code_points[2], combined) & 0xFFFFu; #else return (intHashCRC32(combined) ^ intHashCRC32(code_points[2])) & 0xFFFFu; #endif From be16e0bce78bc998978726545533931340b56743 Mon Sep 17 00:00:00 2001 From: Suzy Wang Date: Thu, 1 Dec 2022 07:43:32 -0800 Subject: [PATCH 02/88] Include crc32_s390x as a submodule library --- .gitmodules | 3 + contrib/CMakeLists.txt | 4 + contrib/crc32-s390x | 1 + contrib/crc32-s390x-cmake/CMakeLists.txt | 19 + src/Common/HashTable/Hash.h | 539 +---------------------- 5 files changed, 33 insertions(+), 533 deletions(-) create mode 160000 contrib/crc32-s390x create mode 100644 contrib/crc32-s390x-cmake/CMakeLists.txt diff --git a/.gitmodules b/.gitmodules index 293029ad171..4260e7c5d9d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -290,3 +290,6 @@ [submodule "contrib/morton-nd"] path = contrib/morton-nd url = https://github.com/morton-nd/morton-nd +[submodule "contrib/crc32-s390x"] + path = contrib/crc32-s390x + url = https://github.com/linux-on-ibm-z/crc32-s390x.git diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 8ebd4ab55d3..b832490434a 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -167,6 +167,10 @@ add_contrib (c-ares-cmake c-ares) add_contrib (qpl-cmake qpl) add_contrib (morton-nd-cmake morton-nd) +if (ARCH_S390X) + add_contrib(crc32-s390x-cmake crc32-s390x) +endif() + add_contrib(annoy-cmake annoy) # Put all targets defined here and in subdirectories under "contrib/" folders in GUI-based IDEs. diff --git a/contrib/crc32-s390x b/contrib/crc32-s390x new file mode 160000 index 00000000000..30980583bf9 --- /dev/null +++ b/contrib/crc32-s390x @@ -0,0 +1 @@ +Subproject commit 30980583bf9ed3fa193abb83a1849705ff457f70 diff --git a/contrib/crc32-s390x-cmake/CMakeLists.txt b/contrib/crc32-s390x-cmake/CMakeLists.txt new file mode 100644 index 00000000000..3df9474c409 --- /dev/null +++ b/contrib/crc32-s390x-cmake/CMakeLists.txt @@ -0,0 +1,19 @@ +if(ARCH_S390X) + option (ENABLE_CRC32_S390X "Enable crc32 on s390x platform" ON) +endif() + +if (NOT ENABLE_CRC32_S390X) + return() +endif() + +set(CRC32_S390X_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/crc32-s390x) +set(CRC32_S390X_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/crc32-s390x) + +set(CRC32_SRCS "${CRC32_S390X_SOURCE_DIR}/crc32-s390x.c") + +add_library(_crc32_s390x ${CRC32_SRCS}) + +target_include_directories(_crc32_s390x SYSTEM PUBLIC "${CRC32_S390X_INCLUDE_DIR}") +target_compile_definitions(_crc32_s390x PUBLIC) + +add_library(ch_contrib::crc32_s390x ALIAS _crc32_s390x) \ No newline at end of file diff --git a/src/Common/HashTable/Hash.h b/src/Common/HashTable/Hash.h index 758286638ca..6b6af7e125d 100644 --- a/src/Common/HashTable/Hash.h +++ b/src/Common/HashTable/Hash.h @@ -49,548 +49,21 @@ inline DB::UInt64 intHash64(DB::UInt64 x) #endif #if defined(__s390x__) && __BYTE_ORDER__==__ORDER_BIG_ENDIAN__ -static const unsigned int __attribute__((aligned(128))) crc32table_be[8][256] = { - { - 0x00000000, 0x04c11db7, 0x09823b6e, 0x0d4326d9, - 0x130476dc, 0x17c56b6b, 0x1a864db2, 0x1e475005, - 0x2608edb8, 0x22c9f00f, 0x2f8ad6d6, 0x2b4bcb61, - 0x350c9b64, 0x31cd86d3, 0x3c8ea00a, 0x384fbdbd, - 0x4c11db70, 0x48d0c6c7, 0x4593e01e, 0x4152fda9, - 0x5f15adac, 0x5bd4b01b, 0x569796c2, 0x52568b75, - 0x6a1936c8, 0x6ed82b7f, 0x639b0da6, 0x675a1011, - 0x791d4014, 0x7ddc5da3, 0x709f7b7a, 0x745e66cd, - 0x9823b6e0, 0x9ce2ab57, 0x91a18d8e, 0x95609039, - 0x8b27c03c, 0x8fe6dd8b, 0x82a5fb52, 0x8664e6e5, - 0xbe2b5b58, 0xbaea46ef, 0xb7a96036, 0xb3687d81, - 0xad2f2d84, 0xa9ee3033, 0xa4ad16ea, 0xa06c0b5d, - 0xd4326d90, 0xd0f37027, 0xddb056fe, 0xd9714b49, - 0xc7361b4c, 0xc3f706fb, 0xceb42022, 0xca753d95, - 0xf23a8028, 0xf6fb9d9f, 0xfbb8bb46, 0xff79a6f1, - 0xe13ef6f4, 0xe5ffeb43, 0xe8bccd9a, 0xec7dd02d, - 0x34867077, 0x30476dc0, 0x3d044b19, 0x39c556ae, - 0x278206ab, 0x23431b1c, 0x2e003dc5, 0x2ac12072, - 0x128e9dcf, 0x164f8078, 0x1b0ca6a1, 0x1fcdbb16, - 0x018aeb13, 0x054bf6a4, 0x0808d07d, 0x0cc9cdca, - 0x7897ab07, 0x7c56b6b0, 0x71159069, 0x75d48dde, - 0x6b93dddb, 0x6f52c06c, 0x6211e6b5, 0x66d0fb02, - 0x5e9f46bf, 0x5a5e5b08, 0x571d7dd1, 0x53dc6066, - 0x4d9b3063, 0x495a2dd4, 0x44190b0d, 0x40d816ba, - 0xaca5c697, 0xa864db20, 0xa527fdf9, 0xa1e6e04e, - 0xbfa1b04b, 0xbb60adfc, 0xb6238b25, 0xb2e29692, - 0x8aad2b2f, 0x8e6c3698, 0x832f1041, 0x87ee0df6, - 0x99a95df3, 0x9d684044, 0x902b669d, 0x94ea7b2a, - 0xe0b41de7, 0xe4750050, 0xe9362689, 0xedf73b3e, - 0xf3b06b3b, 0xf771768c, 0xfa325055, 0xfef34de2, - 0xc6bcf05f, 0xc27dede8, 0xcf3ecb31, 0xcbffd686, - 0xd5b88683, 0xd1799b34, 0xdc3abded, 0xd8fba05a, - 0x690ce0ee, 0x6dcdfd59, 0x608edb80, 0x644fc637, - 0x7a089632, 0x7ec98b85, 0x738aad5c, 0x774bb0eb, - 0x4f040d56, 0x4bc510e1, 0x46863638, 0x42472b8f, - 0x5c007b8a, 0x58c1663d, 0x558240e4, 0x51435d53, - 0x251d3b9e, 0x21dc2629, 0x2c9f00f0, 0x285e1d47, - 0x36194d42, 0x32d850f5, 0x3f9b762c, 0x3b5a6b9b, - 0x0315d626, 0x07d4cb91, 0x0a97ed48, 0x0e56f0ff, - 0x1011a0fa, 0x14d0bd4d, 0x19939b94, 0x1d528623, - 0xf12f560e, 0xf5ee4bb9, 0xf8ad6d60, 0xfc6c70d7, - 0xe22b20d2, 0xe6ea3d65, 0xeba91bbc, 0xef68060b, - 0xd727bbb6, 0xd3e6a601, 0xdea580d8, 0xda649d6f, - 0xc423cd6a, 0xc0e2d0dd, 0xcda1f604, 0xc960ebb3, - 0xbd3e8d7e, 0xb9ff90c9, 0xb4bcb610, 0xb07daba7, - 0xae3afba2, 0xaafbe615, 0xa7b8c0cc, 0xa379dd7b, - 0x9b3660c6, 0x9ff77d71, 0x92b45ba8, 0x9675461f, - 0x8832161a, 0x8cf30bad, 0x81b02d74, 0x857130c3, - 0x5d8a9099, 0x594b8d2e, 0x5408abf7, 0x50c9b640, - 0x4e8ee645, 0x4a4ffbf2, 0x470cdd2b, 0x43cdc09c, - 0x7b827d21, 0x7f436096, 0x7200464f, 0x76c15bf8, - 0x68860bfd, 0x6c47164a, 0x61043093, 0x65c52d24, - 0x119b4be9, 0x155a565e, 0x18197087, 0x1cd86d30, - 0x029f3d35, 0x065e2082, 0x0b1d065b, 0x0fdc1bec, - 0x3793a651, 0x3352bbe6, 0x3e119d3f, 0x3ad08088, - 0x2497d08d, 0x2056cd3a, 0x2d15ebe3, 0x29d4f654, - 0xc5a92679, 0xc1683bce, 0xcc2b1d17, 0xc8ea00a0, - 0xd6ad50a5, 0xd26c4d12, 0xdf2f6bcb, 0xdbee767c, - 0xe3a1cbc1, 0xe760d676, 0xea23f0af, 0xeee2ed18, - 0xf0a5bd1d, 0xf464a0aa, 0xf9278673, 0xfde69bc4, - 0x89b8fd09, 0x8d79e0be, 0x803ac667, 0x84fbdbd0, - 0x9abc8bd5, 0x9e7d9662, 0x933eb0bb, 0x97ffad0c, - 0xafb010b1, 0xab710d06, 0xa6322bdf, 0xa2f33668, - 0xbcb4666d, 0xb8757bda, 0xb5365d03, 0xb1f740b4 - },{ - 0x00000000, 0xd219c1dc, 0xa0f29e0f, 0x72eb5fd3, - 0x452421a9, 0x973de075, 0xe5d6bfa6, 0x37cf7e7a, - 0x8a484352, 0x5851828e, 0x2abadd5d, 0xf8a31c81, - 0xcf6c62fb, 0x1d75a327, 0x6f9efcf4, 0xbd873d28, - 0x10519b13, 0xc2485acf, 0xb0a3051c, 0x62bac4c0, - 0x5575baba, 0x876c7b66, 0xf58724b5, 0x279ee569, - 0x9a19d841, 0x4800199d, 0x3aeb464e, 0xe8f28792, - 0xdf3df9e8, 0x0d243834, 0x7fcf67e7, 0xadd6a63b, - 0x20a33626, 0xf2baf7fa, 0x8051a829, 0x524869f5, - 0x6587178f, 0xb79ed653, 0xc5758980, 0x176c485c, - 0xaaeb7574, 0x78f2b4a8, 0x0a19eb7b, 0xd8002aa7, - 0xefcf54dd, 0x3dd69501, 0x4f3dcad2, 0x9d240b0e, - 0x30f2ad35, 0xe2eb6ce9, 0x9000333a, 0x4219f2e6, - 0x75d68c9c, 0xa7cf4d40, 0xd5241293, 0x073dd34f, - 0xbabaee67, 0x68a32fbb, 0x1a487068, 0xc851b1b4, - 0xff9ecfce, 0x2d870e12, 0x5f6c51c1, 0x8d75901d, - 0x41466c4c, 0x935fad90, 0xe1b4f243, 0x33ad339f, - 0x04624de5, 0xd67b8c39, 0xa490d3ea, 0x76891236, - 0xcb0e2f1e, 0x1917eec2, 0x6bfcb111, 0xb9e570cd, - 0x8e2a0eb7, 0x5c33cf6b, 0x2ed890b8, 0xfcc15164, - 0x5117f75f, 0x830e3683, 0xf1e56950, 0x23fca88c, - 0x1433d6f6, 0xc62a172a, 0xb4c148f9, 0x66d88925, - 0xdb5fb40d, 0x094675d1, 0x7bad2a02, 0xa9b4ebde, - 0x9e7b95a4, 0x4c625478, 0x3e890bab, 0xec90ca77, - 0x61e55a6a, 0xb3fc9bb6, 0xc117c465, 0x130e05b9, - 0x24c17bc3, 0xf6d8ba1f, 0x8433e5cc, 0x562a2410, - 0xebad1938, 0x39b4d8e4, 0x4b5f8737, 0x994646eb, - 0xae893891, 0x7c90f94d, 0x0e7ba69e, 0xdc626742, - 0x71b4c179, 0xa3ad00a5, 0xd1465f76, 0x035f9eaa, - 0x3490e0d0, 0xe689210c, 0x94627edf, 0x467bbf03, - 0xfbfc822b, 0x29e543f7, 0x5b0e1c24, 0x8917ddf8, - 0xbed8a382, 0x6cc1625e, 0x1e2a3d8d, 0xcc33fc51, - 0x828cd898, 0x50951944, 0x227e4697, 0xf067874b, - 0xc7a8f931, 0x15b138ed, 0x675a673e, 0xb543a6e2, - 0x08c49bca, 0xdadd5a16, 0xa83605c5, 0x7a2fc419, - 0x4de0ba63, 0x9ff97bbf, 0xed12246c, 0x3f0be5b0, - 0x92dd438b, 0x40c48257, 0x322fdd84, 0xe0361c58, - 0xd7f96222, 0x05e0a3fe, 0x770bfc2d, 0xa5123df1, - 0x189500d9, 0xca8cc105, 0xb8679ed6, 0x6a7e5f0a, - 0x5db12170, 0x8fa8e0ac, 0xfd43bf7f, 0x2f5a7ea3, - 0xa22feebe, 0x70362f62, 0x02dd70b1, 0xd0c4b16d, - 0xe70bcf17, 0x35120ecb, 0x47f95118, 0x95e090c4, - 0x2867adec, 0xfa7e6c30, 0x889533e3, 0x5a8cf23f, - 0x6d438c45, 0xbf5a4d99, 0xcdb1124a, 0x1fa8d396, - 0xb27e75ad, 0x6067b471, 0x128ceba2, 0xc0952a7e, - 0xf75a5404, 0x254395d8, 0x57a8ca0b, 0x85b10bd7, - 0x383636ff, 0xea2ff723, 0x98c4a8f0, 0x4add692c, - 0x7d121756, 0xaf0bd68a, 0xdde08959, 0x0ff94885, - 0xc3cab4d4, 0x11d37508, 0x63382adb, 0xb121eb07, - 0x86ee957d, 0x54f754a1, 0x261c0b72, 0xf405caae, - 0x4982f786, 0x9b9b365a, 0xe9706989, 0x3b69a855, - 0x0ca6d62f, 0xdebf17f3, 0xac544820, 0x7e4d89fc, - 0xd39b2fc7, 0x0182ee1b, 0x7369b1c8, 0xa1707014, - 0x96bf0e6e, 0x44a6cfb2, 0x364d9061, 0xe45451bd, - 0x59d36c95, 0x8bcaad49, 0xf921f29a, 0x2b383346, - 0x1cf74d3c, 0xceee8ce0, 0xbc05d333, 0x6e1c12ef, - 0xe36982f2, 0x3170432e, 0x439b1cfd, 0x9182dd21, - 0xa64da35b, 0x74546287, 0x06bf3d54, 0xd4a6fc88, - 0x6921c1a0, 0xbb38007c, 0xc9d35faf, 0x1bca9e73, - 0x2c05e009, 0xfe1c21d5, 0x8cf77e06, 0x5eeebfda, - 0xf33819e1, 0x2121d83d, 0x53ca87ee, 0x81d34632, - 0xb61c3848, 0x6405f994, 0x16eea647, 0xc4f7679b, - 0x79705ab3, 0xab699b6f, 0xd982c4bc, 0x0b9b0560, - 0x3c547b1a, 0xee4dbac6, 0x9ca6e515, 0x4ebf24c9 - },{ - 0x00000000, 0x01d8ac87, 0x03b1590e, 0x0269f589, - 0x0762b21c, 0x06ba1e9b, 0x04d3eb12, 0x050b4795, - 0x0ec56438, 0x0f1dc8bf, 0x0d743d36, 0x0cac91b1, - 0x09a7d624, 0x087f7aa3, 0x0a168f2a, 0x0bce23ad, - 0x1d8ac870, 0x1c5264f7, 0x1e3b917e, 0x1fe33df9, - 0x1ae87a6c, 0x1b30d6eb, 0x19592362, 0x18818fe5, - 0x134fac48, 0x129700cf, 0x10fef546, 0x112659c1, - 0x142d1e54, 0x15f5b2d3, 0x179c475a, 0x1644ebdd, - 0x3b1590e0, 0x3acd3c67, 0x38a4c9ee, 0x397c6569, - 0x3c7722fc, 0x3daf8e7b, 0x3fc67bf2, 0x3e1ed775, - 0x35d0f4d8, 0x3408585f, 0x3661add6, 0x37b90151, - 0x32b246c4, 0x336aea43, 0x31031fca, 0x30dbb34d, - 0x269f5890, 0x2747f417, 0x252e019e, 0x24f6ad19, - 0x21fdea8c, 0x2025460b, 0x224cb382, 0x23941f05, - 0x285a3ca8, 0x2982902f, 0x2beb65a6, 0x2a33c921, - 0x2f388eb4, 0x2ee02233, 0x2c89d7ba, 0x2d517b3d, - 0x762b21c0, 0x77f38d47, 0x759a78ce, 0x7442d449, - 0x714993dc, 0x70913f5b, 0x72f8cad2, 0x73206655, - 0x78ee45f8, 0x7936e97f, 0x7b5f1cf6, 0x7a87b071, - 0x7f8cf7e4, 0x7e545b63, 0x7c3daeea, 0x7de5026d, - 0x6ba1e9b0, 0x6a794537, 0x6810b0be, 0x69c81c39, - 0x6cc35bac, 0x6d1bf72b, 0x6f7202a2, 0x6eaaae25, - 0x65648d88, 0x64bc210f, 0x66d5d486, 0x670d7801, - 0x62063f94, 0x63de9313, 0x61b7669a, 0x606fca1d, - 0x4d3eb120, 0x4ce61da7, 0x4e8fe82e, 0x4f5744a9, - 0x4a5c033c, 0x4b84afbb, 0x49ed5a32, 0x4835f6b5, - 0x43fbd518, 0x4223799f, 0x404a8c16, 0x41922091, - 0x44996704, 0x4541cb83, 0x47283e0a, 0x46f0928d, - 0x50b47950, 0x516cd5d7, 0x5305205e, 0x52dd8cd9, - 0x57d6cb4c, 0x560e67cb, 0x54679242, 0x55bf3ec5, - 0x5e711d68, 0x5fa9b1ef, 0x5dc04466, 0x5c18e8e1, - 0x5913af74, 0x58cb03f3, 0x5aa2f67a, 0x5b7a5afd, - 0xec564380, 0xed8eef07, 0xefe71a8e, 0xee3fb609, - 0xeb34f19c, 0xeaec5d1b, 0xe885a892, 0xe95d0415, - 0xe29327b8, 0xe34b8b3f, 0xe1227eb6, 0xe0fad231, - 0xe5f195a4, 0xe4293923, 0xe640ccaa, 0xe798602d, - 0xf1dc8bf0, 0xf0042777, 0xf26dd2fe, 0xf3b57e79, - 0xf6be39ec, 0xf766956b, 0xf50f60e2, 0xf4d7cc65, - 0xff19efc8, 0xfec1434f, 0xfca8b6c6, 0xfd701a41, - 0xf87b5dd4, 0xf9a3f153, 0xfbca04da, 0xfa12a85d, - 0xd743d360, 0xd69b7fe7, 0xd4f28a6e, 0xd52a26e9, - 0xd021617c, 0xd1f9cdfb, 0xd3903872, 0xd24894f5, - 0xd986b758, 0xd85e1bdf, 0xda37ee56, 0xdbef42d1, - 0xdee40544, 0xdf3ca9c3, 0xdd555c4a, 0xdc8df0cd, - 0xcac91b10, 0xcb11b797, 0xc978421e, 0xc8a0ee99, - 0xcdaba90c, 0xcc73058b, 0xce1af002, 0xcfc25c85, - 0xc40c7f28, 0xc5d4d3af, 0xc7bd2626, 0xc6658aa1, - 0xc36ecd34, 0xc2b661b3, 0xc0df943a, 0xc10738bd, - 0x9a7d6240, 0x9ba5cec7, 0x99cc3b4e, 0x981497c9, - 0x9d1fd05c, 0x9cc77cdb, 0x9eae8952, 0x9f7625d5, - 0x94b80678, 0x9560aaff, 0x97095f76, 0x96d1f3f1, - 0x93dab464, 0x920218e3, 0x906bed6a, 0x91b341ed, - 0x87f7aa30, 0x862f06b7, 0x8446f33e, 0x859e5fb9, - 0x8095182c, 0x814db4ab, 0x83244122, 0x82fceda5, - 0x8932ce08, 0x88ea628f, 0x8a839706, 0x8b5b3b81, - 0x8e507c14, 0x8f88d093, 0x8de1251a, 0x8c39899d, - 0xa168f2a0, 0xa0b05e27, 0xa2d9abae, 0xa3010729, - 0xa60a40bc, 0xa7d2ec3b, 0xa5bb19b2, 0xa463b535, - 0xafad9698, 0xae753a1f, 0xac1ccf96, 0xadc46311, - 0xa8cf2484, 0xa9178803, 0xab7e7d8a, 0xaaa6d10d, - 0xbce23ad0, 0xbd3a9657, 0xbf5363de, 0xbe8bcf59, - 0xbb8088cc, 0xba58244b, 0xb831d1c2, 0xb9e97d45, - 0xb2275ee8, 0xb3fff26f, 0xb19607e6, 0xb04eab61, - 0xb545ecf4, 0xb49d4073, 0xb6f4b5fa, 0xb72c197d - },{ - 0x00000000, 0xdc6d9ab7, 0xbc1a28d9, 0x6077b26e, - 0x7cf54c05, 0xa098d6b2, 0xc0ef64dc, 0x1c82fe6b, - 0xf9ea980a, 0x258702bd, 0x45f0b0d3, 0x999d2a64, - 0x851fd40f, 0x59724eb8, 0x3905fcd6, 0xe5686661, - 0xf7142da3, 0x2b79b714, 0x4b0e057a, 0x97639fcd, - 0x8be161a6, 0x578cfb11, 0x37fb497f, 0xeb96d3c8, - 0x0efeb5a9, 0xd2932f1e, 0xb2e49d70, 0x6e8907c7, - 0x720bf9ac, 0xae66631b, 0xce11d175, 0x127c4bc2, - 0xeae946f1, 0x3684dc46, 0x56f36e28, 0x8a9ef49f, - 0x961c0af4, 0x4a719043, 0x2a06222d, 0xf66bb89a, - 0x1303defb, 0xcf6e444c, 0xaf19f622, 0x73746c95, - 0x6ff692fe, 0xb39b0849, 0xd3ecba27, 0x0f812090, - 0x1dfd6b52, 0xc190f1e5, 0xa1e7438b, 0x7d8ad93c, - 0x61082757, 0xbd65bde0, 0xdd120f8e, 0x017f9539, - 0xe417f358, 0x387a69ef, 0x580ddb81, 0x84604136, - 0x98e2bf5d, 0x448f25ea, 0x24f89784, 0xf8950d33, - 0xd1139055, 0x0d7e0ae2, 0x6d09b88c, 0xb164223b, - 0xade6dc50, 0x718b46e7, 0x11fcf489, 0xcd916e3e, - 0x28f9085f, 0xf49492e8, 0x94e32086, 0x488eba31, - 0x540c445a, 0x8861deed, 0xe8166c83, 0x347bf634, - 0x2607bdf6, 0xfa6a2741, 0x9a1d952f, 0x46700f98, - 0x5af2f1f3, 0x869f6b44, 0xe6e8d92a, 0x3a85439d, - 0xdfed25fc, 0x0380bf4b, 0x63f70d25, 0xbf9a9792, - 0xa31869f9, 0x7f75f34e, 0x1f024120, 0xc36fdb97, - 0x3bfad6a4, 0xe7974c13, 0x87e0fe7d, 0x5b8d64ca, - 0x470f9aa1, 0x9b620016, 0xfb15b278, 0x277828cf, - 0xc2104eae, 0x1e7dd419, 0x7e0a6677, 0xa267fcc0, - 0xbee502ab, 0x6288981c, 0x02ff2a72, 0xde92b0c5, - 0xcceefb07, 0x108361b0, 0x70f4d3de, 0xac994969, - 0xb01bb702, 0x6c762db5, 0x0c019fdb, 0xd06c056c, - 0x3504630d, 0xe969f9ba, 0x891e4bd4, 0x5573d163, - 0x49f12f08, 0x959cb5bf, 0xf5eb07d1, 0x29869d66, - 0xa6e63d1d, 0x7a8ba7aa, 0x1afc15c4, 0xc6918f73, - 0xda137118, 0x067eebaf, 0x660959c1, 0xba64c376, - 0x5f0ca517, 0x83613fa0, 0xe3168dce, 0x3f7b1779, - 0x23f9e912, 0xff9473a5, 0x9fe3c1cb, 0x438e5b7c, - 0x51f210be, 0x8d9f8a09, 0xede83867, 0x3185a2d0, - 0x2d075cbb, 0xf16ac60c, 0x911d7462, 0x4d70eed5, - 0xa81888b4, 0x74751203, 0x1402a06d, 0xc86f3ada, - 0xd4edc4b1, 0x08805e06, 0x68f7ec68, 0xb49a76df, - 0x4c0f7bec, 0x9062e15b, 0xf0155335, 0x2c78c982, - 0x30fa37e9, 0xec97ad5e, 0x8ce01f30, 0x508d8587, - 0xb5e5e3e6, 0x69887951, 0x09ffcb3f, 0xd5925188, - 0xc910afe3, 0x157d3554, 0x750a873a, 0xa9671d8d, - 0xbb1b564f, 0x6776ccf8, 0x07017e96, 0xdb6ce421, - 0xc7ee1a4a, 0x1b8380fd, 0x7bf43293, 0xa799a824, - 0x42f1ce45, 0x9e9c54f2, 0xfeebe69c, 0x22867c2b, - 0x3e048240, 0xe26918f7, 0x821eaa99, 0x5e73302e, - 0x77f5ad48, 0xab9837ff, 0xcbef8591, 0x17821f26, - 0x0b00e14d, 0xd76d7bfa, 0xb71ac994, 0x6b775323, - 0x8e1f3542, 0x5272aff5, 0x32051d9b, 0xee68872c, - 0xf2ea7947, 0x2e87e3f0, 0x4ef0519e, 0x929dcb29, - 0x80e180eb, 0x5c8c1a5c, 0x3cfba832, 0xe0963285, - 0xfc14ccee, 0x20795659, 0x400ee437, 0x9c637e80, - 0x790b18e1, 0xa5668256, 0xc5113038, 0x197caa8f, - 0x05fe54e4, 0xd993ce53, 0xb9e47c3d, 0x6589e68a, - 0x9d1cebb9, 0x4171710e, 0x2106c360, 0xfd6b59d7, - 0xe1e9a7bc, 0x3d843d0b, 0x5df38f65, 0x819e15d2, - 0x64f673b3, 0xb89be904, 0xd8ec5b6a, 0x0481c1dd, - 0x18033fb6, 0xc46ea501, 0xa419176f, 0x78748dd8, - 0x6a08c61a, 0xb6655cad, 0xd612eec3, 0x0a7f7474, - 0x16fd8a1f, 0xca9010a8, 0xaae7a2c6, 0x768a3871, - 0x93e25e10, 0x4f8fc4a7, 0x2ff876c9, 0xf395ec7e, - 0xef171215, 0x337a88a2, 0x530d3acc, 0x8f60a07b - },{ - 0x00000000, 0x490d678d, 0x921acf1a, 0xdb17a897, - 0x20f48383, 0x69f9e40e, 0xb2ee4c99, 0xfbe32b14, - 0x41e90706, 0x08e4608b, 0xd3f3c81c, 0x9afeaf91, - 0x611d8485, 0x2810e308, 0xf3074b9f, 0xba0a2c12, - 0x83d20e0c, 0xcadf6981, 0x11c8c116, 0x58c5a69b, - 0xa3268d8f, 0xea2bea02, 0x313c4295, 0x78312518, - 0xc23b090a, 0x8b366e87, 0x5021c610, 0x192ca19d, - 0xe2cf8a89, 0xabc2ed04, 0x70d54593, 0x39d8221e, - 0x036501af, 0x4a686622, 0x917fceb5, 0xd872a938, - 0x2391822c, 0x6a9ce5a1, 0xb18b4d36, 0xf8862abb, - 0x428c06a9, 0x0b816124, 0xd096c9b3, 0x999bae3e, - 0x6278852a, 0x2b75e2a7, 0xf0624a30, 0xb96f2dbd, - 0x80b70fa3, 0xc9ba682e, 0x12adc0b9, 0x5ba0a734, - 0xa0438c20, 0xe94eebad, 0x3259433a, 0x7b5424b7, - 0xc15e08a5, 0x88536f28, 0x5344c7bf, 0x1a49a032, - 0xe1aa8b26, 0xa8a7ecab, 0x73b0443c, 0x3abd23b1, - 0x06ca035e, 0x4fc764d3, 0x94d0cc44, 0xddddabc9, - 0x263e80dd, 0x6f33e750, 0xb4244fc7, 0xfd29284a, - 0x47230458, 0x0e2e63d5, 0xd539cb42, 0x9c34accf, - 0x67d787db, 0x2edae056, 0xf5cd48c1, 0xbcc02f4c, - 0x85180d52, 0xcc156adf, 0x1702c248, 0x5e0fa5c5, - 0xa5ec8ed1, 0xece1e95c, 0x37f641cb, 0x7efb2646, - 0xc4f10a54, 0x8dfc6dd9, 0x56ebc54e, 0x1fe6a2c3, - 0xe40589d7, 0xad08ee5a, 0x761f46cd, 0x3f122140, - 0x05af02f1, 0x4ca2657c, 0x97b5cdeb, 0xdeb8aa66, - 0x255b8172, 0x6c56e6ff, 0xb7414e68, 0xfe4c29e5, - 0x444605f7, 0x0d4b627a, 0xd65ccaed, 0x9f51ad60, - 0x64b28674, 0x2dbfe1f9, 0xf6a8496e, 0xbfa52ee3, - 0x867d0cfd, 0xcf706b70, 0x1467c3e7, 0x5d6aa46a, - 0xa6898f7e, 0xef84e8f3, 0x34934064, 0x7d9e27e9, - 0xc7940bfb, 0x8e996c76, 0x558ec4e1, 0x1c83a36c, - 0xe7608878, 0xae6deff5, 0x757a4762, 0x3c7720ef, - 0x0d9406bc, 0x44996131, 0x9f8ec9a6, 0xd683ae2b, - 0x2d60853f, 0x646de2b2, 0xbf7a4a25, 0xf6772da8, - 0x4c7d01ba, 0x05706637, 0xde67cea0, 0x976aa92d, - 0x6c898239, 0x2584e5b4, 0xfe934d23, 0xb79e2aae, - 0x8e4608b0, 0xc74b6f3d, 0x1c5cc7aa, 0x5551a027, - 0xaeb28b33, 0xe7bfecbe, 0x3ca84429, 0x75a523a4, - 0xcfaf0fb6, 0x86a2683b, 0x5db5c0ac, 0x14b8a721, - 0xef5b8c35, 0xa656ebb8, 0x7d41432f, 0x344c24a2, - 0x0ef10713, 0x47fc609e, 0x9cebc809, 0xd5e6af84, - 0x2e058490, 0x6708e31d, 0xbc1f4b8a, 0xf5122c07, - 0x4f180015, 0x06156798, 0xdd02cf0f, 0x940fa882, - 0x6fec8396, 0x26e1e41b, 0xfdf64c8c, 0xb4fb2b01, - 0x8d23091f, 0xc42e6e92, 0x1f39c605, 0x5634a188, - 0xadd78a9c, 0xe4daed11, 0x3fcd4586, 0x76c0220b, - 0xccca0e19, 0x85c76994, 0x5ed0c103, 0x17dda68e, - 0xec3e8d9a, 0xa533ea17, 0x7e244280, 0x3729250d, - 0x0b5e05e2, 0x4253626f, 0x9944caf8, 0xd049ad75, - 0x2baa8661, 0x62a7e1ec, 0xb9b0497b, 0xf0bd2ef6, - 0x4ab702e4, 0x03ba6569, 0xd8adcdfe, 0x91a0aa73, - 0x6a438167, 0x234ee6ea, 0xf8594e7d, 0xb15429f0, - 0x888c0bee, 0xc1816c63, 0x1a96c4f4, 0x539ba379, - 0xa878886d, 0xe175efe0, 0x3a624777, 0x736f20fa, - 0xc9650ce8, 0x80686b65, 0x5b7fc3f2, 0x1272a47f, - 0xe9918f6b, 0xa09ce8e6, 0x7b8b4071, 0x328627fc, - 0x083b044d, 0x413663c0, 0x9a21cb57, 0xd32cacda, - 0x28cf87ce, 0x61c2e043, 0xbad548d4, 0xf3d82f59, - 0x49d2034b, 0x00df64c6, 0xdbc8cc51, 0x92c5abdc, - 0x692680c8, 0x202be745, 0xfb3c4fd2, 0xb231285f, - 0x8be90a41, 0xc2e46dcc, 0x19f3c55b, 0x50fea2d6, - 0xab1d89c2, 0xe210ee4f, 0x390746d8, 0x700a2155, - 0xca000d47, 0x830d6aca, 0x581ac25d, 0x1117a5d0, - 0xeaf48ec4, 0xa3f9e949, 0x78ee41de, 0x31e32653 - },{ - 0x00000000, 0x1b280d78, 0x36501af0, 0x2d781788, - 0x6ca035e0, 0x77883898, 0x5af02f10, 0x41d82268, - 0xd9406bc0, 0xc26866b8, 0xef107130, 0xf4387c48, - 0xb5e05e20, 0xaec85358, 0x83b044d0, 0x989849a8, - 0xb641ca37, 0xad69c74f, 0x8011d0c7, 0x9b39ddbf, - 0xdae1ffd7, 0xc1c9f2af, 0xecb1e527, 0xf799e85f, - 0x6f01a1f7, 0x7429ac8f, 0x5951bb07, 0x4279b67f, - 0x03a19417, 0x1889996f, 0x35f18ee7, 0x2ed9839f, - 0x684289d9, 0x736a84a1, 0x5e129329, 0x453a9e51, - 0x04e2bc39, 0x1fcab141, 0x32b2a6c9, 0x299aabb1, - 0xb102e219, 0xaa2aef61, 0x8752f8e9, 0x9c7af591, - 0xdda2d7f9, 0xc68ada81, 0xebf2cd09, 0xf0dac071, - 0xde0343ee, 0xc52b4e96, 0xe853591e, 0xf37b5466, - 0xb2a3760e, 0xa98b7b76, 0x84f36cfe, 0x9fdb6186, - 0x0743282e, 0x1c6b2556, 0x311332de, 0x2a3b3fa6, - 0x6be31dce, 0x70cb10b6, 0x5db3073e, 0x469b0a46, - 0xd08513b2, 0xcbad1eca, 0xe6d50942, 0xfdfd043a, - 0xbc252652, 0xa70d2b2a, 0x8a753ca2, 0x915d31da, - 0x09c57872, 0x12ed750a, 0x3f956282, 0x24bd6ffa, - 0x65654d92, 0x7e4d40ea, 0x53355762, 0x481d5a1a, - 0x66c4d985, 0x7decd4fd, 0x5094c375, 0x4bbcce0d, - 0x0a64ec65, 0x114ce11d, 0x3c34f695, 0x271cfbed, - 0xbf84b245, 0xa4acbf3d, 0x89d4a8b5, 0x92fca5cd, - 0xd32487a5, 0xc80c8add, 0xe5749d55, 0xfe5c902d, - 0xb8c79a6b, 0xa3ef9713, 0x8e97809b, 0x95bf8de3, - 0xd467af8b, 0xcf4fa2f3, 0xe237b57b, 0xf91fb803, - 0x6187f1ab, 0x7aaffcd3, 0x57d7eb5b, 0x4cffe623, - 0x0d27c44b, 0x160fc933, 0x3b77debb, 0x205fd3c3, - 0x0e86505c, 0x15ae5d24, 0x38d64aac, 0x23fe47d4, - 0x622665bc, 0x790e68c4, 0x54767f4c, 0x4f5e7234, - 0xd7c63b9c, 0xccee36e4, 0xe196216c, 0xfabe2c14, - 0xbb660e7c, 0xa04e0304, 0x8d36148c, 0x961e19f4, - 0xa5cb3ad3, 0xbee337ab, 0x939b2023, 0x88b32d5b, - 0xc96b0f33, 0xd243024b, 0xff3b15c3, 0xe41318bb, - 0x7c8b5113, 0x67a35c6b, 0x4adb4be3, 0x51f3469b, - 0x102b64f3, 0x0b03698b, 0x267b7e03, 0x3d53737b, - 0x138af0e4, 0x08a2fd9c, 0x25daea14, 0x3ef2e76c, - 0x7f2ac504, 0x6402c87c, 0x497adff4, 0x5252d28c, - 0xcaca9b24, 0xd1e2965c, 0xfc9a81d4, 0xe7b28cac, - 0xa66aaec4, 0xbd42a3bc, 0x903ab434, 0x8b12b94c, - 0xcd89b30a, 0xd6a1be72, 0xfbd9a9fa, 0xe0f1a482, - 0xa12986ea, 0xba018b92, 0x97799c1a, 0x8c519162, - 0x14c9d8ca, 0x0fe1d5b2, 0x2299c23a, 0x39b1cf42, - 0x7869ed2a, 0x6341e052, 0x4e39f7da, 0x5511faa2, - 0x7bc8793d, 0x60e07445, 0x4d9863cd, 0x56b06eb5, - 0x17684cdd, 0x0c4041a5, 0x2138562d, 0x3a105b55, - 0xa28812fd, 0xb9a01f85, 0x94d8080d, 0x8ff00575, - 0xce28271d, 0xd5002a65, 0xf8783ded, 0xe3503095, - 0x754e2961, 0x6e662419, 0x431e3391, 0x58363ee9, - 0x19ee1c81, 0x02c611f9, 0x2fbe0671, 0x34960b09, - 0xac0e42a1, 0xb7264fd9, 0x9a5e5851, 0x81765529, - 0xc0ae7741, 0xdb867a39, 0xf6fe6db1, 0xedd660c9, - 0xc30fe356, 0xd827ee2e, 0xf55ff9a6, 0xee77f4de, - 0xafafd6b6, 0xb487dbce, 0x99ffcc46, 0x82d7c13e, - 0x1a4f8896, 0x016785ee, 0x2c1f9266, 0x37379f1e, - 0x76efbd76, 0x6dc7b00e, 0x40bfa786, 0x5b97aafe, - 0x1d0ca0b8, 0x0624adc0, 0x2b5cba48, 0x3074b730, - 0x71ac9558, 0x6a849820, 0x47fc8fa8, 0x5cd482d0, - 0xc44ccb78, 0xdf64c600, 0xf21cd188, 0xe934dcf0, - 0xa8ecfe98, 0xb3c4f3e0, 0x9ebce468, 0x8594e910, - 0xab4d6a8f, 0xb06567f7, 0x9d1d707f, 0x86357d07, - 0xc7ed5f6f, 0xdcc55217, 0xf1bd459f, 0xea9548e7, - 0x720d014f, 0x69250c37, 0x445d1bbf, 0x5f7516c7, - 0x1ead34af, 0x058539d7, 0x28fd2e5f, 0x33d52327 - },{ - 0x00000000, 0x4f576811, 0x9eaed022, 0xd1f9b833, - 0x399cbdf3, 0x76cbd5e2, 0xa7326dd1, 0xe86505c0, - 0x73397be6, 0x3c6e13f7, 0xed97abc4, 0xa2c0c3d5, - 0x4aa5c615, 0x05f2ae04, 0xd40b1637, 0x9b5c7e26, - 0xe672f7cc, 0xa9259fdd, 0x78dc27ee, 0x378b4fff, - 0xdfee4a3f, 0x90b9222e, 0x41409a1d, 0x0e17f20c, - 0x954b8c2a, 0xda1ce43b, 0x0be55c08, 0x44b23419, - 0xacd731d9, 0xe38059c8, 0x3279e1fb, 0x7d2e89ea, - 0xc824f22f, 0x87739a3e, 0x568a220d, 0x19dd4a1c, - 0xf1b84fdc, 0xbeef27cd, 0x6f169ffe, 0x2041f7ef, - 0xbb1d89c9, 0xf44ae1d8, 0x25b359eb, 0x6ae431fa, - 0x8281343a, 0xcdd65c2b, 0x1c2fe418, 0x53788c09, - 0x2e5605e3, 0x61016df2, 0xb0f8d5c1, 0xffafbdd0, - 0x17cab810, 0x589dd001, 0x89646832, 0xc6330023, - 0x5d6f7e05, 0x12381614, 0xc3c1ae27, 0x8c96c636, - 0x64f3c3f6, 0x2ba4abe7, 0xfa5d13d4, 0xb50a7bc5, - 0x9488f9e9, 0xdbdf91f8, 0x0a2629cb, 0x457141da, - 0xad14441a, 0xe2432c0b, 0x33ba9438, 0x7cedfc29, - 0xe7b1820f, 0xa8e6ea1e, 0x791f522d, 0x36483a3c, - 0xde2d3ffc, 0x917a57ed, 0x4083efde, 0x0fd487cf, - 0x72fa0e25, 0x3dad6634, 0xec54de07, 0xa303b616, - 0x4b66b3d6, 0x0431dbc7, 0xd5c863f4, 0x9a9f0be5, - 0x01c375c3, 0x4e941dd2, 0x9f6da5e1, 0xd03acdf0, - 0x385fc830, 0x7708a021, 0xa6f11812, 0xe9a67003, - 0x5cac0bc6, 0x13fb63d7, 0xc202dbe4, 0x8d55b3f5, - 0x6530b635, 0x2a67de24, 0xfb9e6617, 0xb4c90e06, - 0x2f957020, 0x60c21831, 0xb13ba002, 0xfe6cc813, - 0x1609cdd3, 0x595ea5c2, 0x88a71df1, 0xc7f075e0, - 0xbadefc0a, 0xf589941b, 0x24702c28, 0x6b274439, - 0x834241f9, 0xcc1529e8, 0x1dec91db, 0x52bbf9ca, - 0xc9e787ec, 0x86b0effd, 0x574957ce, 0x181e3fdf, - 0xf07b3a1f, 0xbf2c520e, 0x6ed5ea3d, 0x2182822c, - 0x2dd0ee65, 0x62878674, 0xb37e3e47, 0xfc295656, - 0x144c5396, 0x5b1b3b87, 0x8ae283b4, 0xc5b5eba5, - 0x5ee99583, 0x11befd92, 0xc04745a1, 0x8f102db0, - 0x67752870, 0x28224061, 0xf9dbf852, 0xb68c9043, - 0xcba219a9, 0x84f571b8, 0x550cc98b, 0x1a5ba19a, - 0xf23ea45a, 0xbd69cc4b, 0x6c907478, 0x23c71c69, - 0xb89b624f, 0xf7cc0a5e, 0x2635b26d, 0x6962da7c, - 0x8107dfbc, 0xce50b7ad, 0x1fa90f9e, 0x50fe678f, - 0xe5f41c4a, 0xaaa3745b, 0x7b5acc68, 0x340da479, - 0xdc68a1b9, 0x933fc9a8, 0x42c6719b, 0x0d91198a, - 0x96cd67ac, 0xd99a0fbd, 0x0863b78e, 0x4734df9f, - 0xaf51da5f, 0xe006b24e, 0x31ff0a7d, 0x7ea8626c, - 0x0386eb86, 0x4cd18397, 0x9d283ba4, 0xd27f53b5, - 0x3a1a5675, 0x754d3e64, 0xa4b48657, 0xebe3ee46, - 0x70bf9060, 0x3fe8f871, 0xee114042, 0xa1462853, - 0x49232d93, 0x06744582, 0xd78dfdb1, 0x98da95a0, - 0xb958178c, 0xf60f7f9d, 0x27f6c7ae, 0x68a1afbf, - 0x80c4aa7f, 0xcf93c26e, 0x1e6a7a5d, 0x513d124c, - 0xca616c6a, 0x8536047b, 0x54cfbc48, 0x1b98d459, - 0xf3fdd199, 0xbcaab988, 0x6d5301bb, 0x220469aa, - 0x5f2ae040, 0x107d8851, 0xc1843062, 0x8ed35873, - 0x66b65db3, 0x29e135a2, 0xf8188d91, 0xb74fe580, - 0x2c139ba6, 0x6344f3b7, 0xb2bd4b84, 0xfdea2395, - 0x158f2655, 0x5ad84e44, 0x8b21f677, 0xc4769e66, - 0x717ce5a3, 0x3e2b8db2, 0xefd23581, 0xa0855d90, - 0x48e05850, 0x07b73041, 0xd64e8872, 0x9919e063, - 0x02459e45, 0x4d12f654, 0x9ceb4e67, 0xd3bc2676, - 0x3bd923b6, 0x748e4ba7, 0xa577f394, 0xea209b85, - 0x970e126f, 0xd8597a7e, 0x09a0c24d, 0x46f7aa5c, - 0xae92af9c, 0xe1c5c78d, 0x303c7fbe, 0x7f6b17af, - 0xe4376989, 0xab600198, 0x7a99b9ab, 0x35ced1ba, - 0xddabd47a, 0x92fcbc6b, 0x43050458, 0x0c526c49 - },{ - 0x00000000, 0x5ba1dcca, 0xb743b994, 0xece2655e, - 0x6a466e9f, 0x31e7b255, 0xdd05d70b, 0x86a40bc1, - 0xd48cdd3e, 0x8f2d01f4, 0x63cf64aa, 0x386eb860, - 0xbecab3a1, 0xe56b6f6b, 0x09890a35, 0x5228d6ff, - 0xadd8a7cb, 0xf6797b01, 0x1a9b1e5f, 0x413ac295, - 0xc79ec954, 0x9c3f159e, 0x70dd70c0, 0x2b7cac0a, - 0x79547af5, 0x22f5a63f, 0xce17c361, 0x95b61fab, - 0x1312146a, 0x48b3c8a0, 0xa451adfe, 0xfff07134, - 0x5f705221, 0x04d18eeb, 0xe833ebb5, 0xb392377f, - 0x35363cbe, 0x6e97e074, 0x8275852a, 0xd9d459e0, - 0x8bfc8f1f, 0xd05d53d5, 0x3cbf368b, 0x671eea41, - 0xe1bae180, 0xba1b3d4a, 0x56f95814, 0x0d5884de, - 0xf2a8f5ea, 0xa9092920, 0x45eb4c7e, 0x1e4a90b4, - 0x98ee9b75, 0xc34f47bf, 0x2fad22e1, 0x740cfe2b, - 0x262428d4, 0x7d85f41e, 0x91679140, 0xcac64d8a, - 0x4c62464b, 0x17c39a81, 0xfb21ffdf, 0xa0802315, - 0xbee0a442, 0xe5417888, 0x09a31dd6, 0x5202c11c, - 0xd4a6cadd, 0x8f071617, 0x63e57349, 0x3844af83, - 0x6a6c797c, 0x31cda5b6, 0xdd2fc0e8, 0x868e1c22, - 0x002a17e3, 0x5b8bcb29, 0xb769ae77, 0xecc872bd, - 0x13380389, 0x4899df43, 0xa47bba1d, 0xffda66d7, - 0x797e6d16, 0x22dfb1dc, 0xce3dd482, 0x959c0848, - 0xc7b4deb7, 0x9c15027d, 0x70f76723, 0x2b56bbe9, - 0xadf2b028, 0xf6536ce2, 0x1ab109bc, 0x4110d576, - 0xe190f663, 0xba312aa9, 0x56d34ff7, 0x0d72933d, - 0x8bd698fc, 0xd0774436, 0x3c952168, 0x6734fda2, - 0x351c2b5d, 0x6ebdf797, 0x825f92c9, 0xd9fe4e03, - 0x5f5a45c2, 0x04fb9908, 0xe819fc56, 0xb3b8209c, - 0x4c4851a8, 0x17e98d62, 0xfb0be83c, 0xa0aa34f6, - 0x260e3f37, 0x7dafe3fd, 0x914d86a3, 0xcaec5a69, - 0x98c48c96, 0xc365505c, 0x2f873502, 0x7426e9c8, - 0xf282e209, 0xa9233ec3, 0x45c15b9d, 0x1e608757, - 0x79005533, 0x22a189f9, 0xce43eca7, 0x95e2306d, - 0x13463bac, 0x48e7e766, 0xa4058238, 0xffa45ef2, - 0xad8c880d, 0xf62d54c7, 0x1acf3199, 0x416eed53, - 0xc7cae692, 0x9c6b3a58, 0x70895f06, 0x2b2883cc, - 0xd4d8f2f8, 0x8f792e32, 0x639b4b6c, 0x383a97a6, - 0xbe9e9c67, 0xe53f40ad, 0x09dd25f3, 0x527cf939, - 0x00542fc6, 0x5bf5f30c, 0xb7179652, 0xecb64a98, - 0x6a124159, 0x31b39d93, 0xdd51f8cd, 0x86f02407, - 0x26700712, 0x7dd1dbd8, 0x9133be86, 0xca92624c, - 0x4c36698d, 0x1797b547, 0xfb75d019, 0xa0d40cd3, - 0xf2fcda2c, 0xa95d06e6, 0x45bf63b8, 0x1e1ebf72, - 0x98bab4b3, 0xc31b6879, 0x2ff90d27, 0x7458d1ed, - 0x8ba8a0d9, 0xd0097c13, 0x3ceb194d, 0x674ac587, - 0xe1eece46, 0xba4f128c, 0x56ad77d2, 0x0d0cab18, - 0x5f247de7, 0x0485a12d, 0xe867c473, 0xb3c618b9, - 0x35621378, 0x6ec3cfb2, 0x8221aaec, 0xd9807626, - 0xc7e0f171, 0x9c412dbb, 0x70a348e5, 0x2b02942f, - 0xada69fee, 0xf6074324, 0x1ae5267a, 0x4144fab0, - 0x136c2c4f, 0x48cdf085, 0xa42f95db, 0xff8e4911, - 0x792a42d0, 0x228b9e1a, 0xce69fb44, 0x95c8278e, - 0x6a3856ba, 0x31998a70, 0xdd7bef2e, 0x86da33e4, - 0x007e3825, 0x5bdfe4ef, 0xb73d81b1, 0xec9c5d7b, - 0xbeb48b84, 0xe515574e, 0x09f73210, 0x5256eeda, - 0xd4f2e51b, 0x8f5339d1, 0x63b15c8f, 0x38108045, - 0x9890a350, 0xc3317f9a, 0x2fd31ac4, 0x7472c60e, - 0xf2d6cdcf, 0xa9771105, 0x4595745b, 0x1e34a891, - 0x4c1c7e6e, 0x17bda2a4, 0xfb5fc7fa, 0xa0fe1b30, - 0x265a10f1, 0x7dfbcc3b, 0x9119a965, 0xcab875af, - 0x3548049b, 0x6ee9d851, 0x820bbd0f, 0xd9aa61c5, - 0x5f0e6a04, 0x04afb6ce, 0xe84dd390, 0xb3ec0f5a, - 0xe1c4d9a5, 0xba65056f, 0x56876031, 0x0d26bcfb, - 0x8b82b73a, 0xd0236bf0, 0x3cc10eae, 0x6760d264 - } -}; -inline unsigned int s390x_crc32_be(unsigned int crc, const unsigned char *buf, size_t len) { - while (len--) - crc = crc32table_be[0][((crc >> 24) ^ *buf++) & 0xFF] ^ (crc << 8); - return crc; -} +#include inline uint32_t s390x_crc32_u8(uint32_t crc, uint8_t v) { - return s390x_crc32_be(crc, reinterpret_cast(&v), sizeof(v)); + return crc32_be(crc, reinterpret_cast(&v), sizeof(v)); } inline uint32_t s390x_crc32_u16(uint32_t crc, uint16_t v) { - return s390x_crc32_be(crc, reinterpret_cast(&v), sizeof(v)); + return crc32_be(crc, reinterpret_cast(&v), sizeof(v)); } inline uint32_t s390x_crc32_u32(uint32_t crc, uint32_t v) { - return s390x_crc32_be(crc, reinterpret_cast(&v), sizeof(v)); + return crc32_be(crc, reinterpret_cast(&v), sizeof(v)); } inline uint64_t s390x_crc32(uint64_t crc, uint64_t v) @@ -599,8 +72,8 @@ inline uint64_t s390x_crc32(uint64_t crc, uint64_t v) uint32_t value_h, value_l; value_h = (v >> 32) & 0xffffffff; value_l = v & 0xffffffff; - _crc = s390x_crc32_be(static_cast(_crc), reinterpret_cast(&value_h), sizeof(uint32_t)); - _crc = s390x_crc32_be(static_cast(_crc), reinterpret_cast(&value_l), sizeof(uint32_t)); + _crc = crc32_be(static_cast(_crc), reinterpret_cast(&value_h), sizeof(uint32_t)); + _crc = crc32_be(static_cast(_crc), reinterpret_cast(&value_l), sizeof(uint32_t)); return _crc; } #endif From 75a298a2753852618b700b112aef57e2e5b446fb Mon Sep 17 00:00:00 2001 From: Suzy Wang Date: Tue, 6 Dec 2022 14:02:22 -0800 Subject: [PATCH 03/88] fix cmake --- contrib/crc32-s390x-cmake/CMakeLists.txt | 12 ++++++++++-- src/CMakeLists.txt | 4 ++++ 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/contrib/crc32-s390x-cmake/CMakeLists.txt b/contrib/crc32-s390x-cmake/CMakeLists.txt index 3df9474c409..25763ebb77d 100644 --- a/contrib/crc32-s390x-cmake/CMakeLists.txt +++ b/contrib/crc32-s390x-cmake/CMakeLists.txt @@ -9,9 +9,17 @@ endif() set(CRC32_S390X_SOURCE_DIR ${ClickHouse_SOURCE_DIR}/contrib/crc32-s390x) set(CRC32_S390X_INCLUDE_DIR ${ClickHouse_SOURCE_DIR}/contrib/crc32-s390x) -set(CRC32_SRCS "${CRC32_S390X_SOURCE_DIR}/crc32-s390x.c") +set(CRC32_SRCS + "${CRC32_S390X_SOURCE_DIR}/crc32-s390x.c" + "${CRC32_S390X_SOURCE_DIR}/crc32be-vx.S" + "${CRC32_S390X_SOURCE_DIR}/crc32le-vx.S" +) -add_library(_crc32_s390x ${CRC32_SRCS}) +set(CRC32_HDRS + "${CRC32_S390X_INCLUDE_DIR}/crc32-s390x.h" +) + +add_library(_crc32_s390x ${CRC32_SRCS} ${CRC32_HDRS}) target_include_directories(_crc32_s390x SYSTEM PUBLIC "${CRC32_S390X_INCLUDE_DIR}") target_compile_definitions(_crc32_s390x PUBLIC) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index ce2cc862b32..642240b9e21 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -387,6 +387,10 @@ if (TARGET ch_contrib::cpuid) target_link_libraries(clickhouse_common_io PRIVATE ch_contrib::cpuid) endif() +if (TARGET ch_contrib::crc32_s390x) + target_link_libraries(clickhouse_common_io PRIVATE ch_contrib::crc32_s390x) +endif() + dbms_target_link_libraries(PUBLIC ch_contrib::abseil_swiss_tables) # Make dbms depend on roaring instead of clickhouse_common_io so that roaring itself can depend on clickhouse_common_io From 556c02d8721c47a88f7c25417aaa7b9d2cf09a9a Mon Sep 17 00:00:00 2001 From: Suzy Wang Date: Tue, 6 Dec 2022 14:08:11 -0800 Subject: [PATCH 04/88] fix cmake --- src/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 642240b9e21..8b984858fcb 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -388,7 +388,7 @@ if (TARGET ch_contrib::cpuid) endif() if (TARGET ch_contrib::crc32_s390x) - target_link_libraries(clickhouse_common_io PRIVATE ch_contrib::crc32_s390x) + target_link_libraries(clickhouse_common_io PUBLIC ch_contrib::crc32_s390x) endif() dbms_target_link_libraries(PUBLIC ch_contrib::abseil_swiss_tables) From 30547d2dcd39a44f91c2b339bbd33a57e3e28830 Mon Sep 17 00:00:00 2001 From: kssenii Date: Fri, 16 Dec 2022 23:57:09 +0100 Subject: [PATCH 05/88] Replace old named collections code for url --- programs/local/LocalServer.cpp | 2 +- programs/server/Server.cpp | 2 +- src/Backups/BackupIO_S3.cpp | 3 +- .../ExternalDictionaryLibraryBridgeHelper.cpp | 2 +- src/CMakeLists.txt | 2 +- .../NamedCollectionConfiguration.cpp | 48 ++++++- .../NamedCollectionConfiguration.h | 24 ++++ .../NamedCollections/NamedCollectionUtils.cpp | 54 +------- .../NamedCollections/NamedCollectionUtils.h | 0 .../NamedCollections/NamedCollections.cpp | 24 +++- .../NamedCollections/NamedCollections.h | 6 +- .../NamedCollections/NamedCollections_fwd.h | 0 .../tests/gtest_named_collections.cpp | 83 +++++++++++- src/Coordination/KeeperSnapshotManagerS3.cpp | 2 +- src/Dictionaries/HTTPDictionarySource.cpp | 6 +- src/Dictionaries/HTTPDictionarySource.h | 2 +- src/Disks/IO/ReadBufferFromWebServer.cpp | 2 +- src/IO/HTTPHeaderEntries.h | 18 +++ src/IO/ReadWriteBufferFromHTTP.h | 16 ++- src/IO/S3/PocoHTTPClient.h | 6 +- src/IO/S3/tests/gtest_aws_s3_client.cpp | 3 +- src/IO/S3Common.cpp | 7 +- src/IO/S3Common.h | 6 +- .../InterpreterAlterNamedCollectionQuery.cpp | 2 +- .../InterpreterCreateNamedCollectionQuery.cpp | 2 +- .../InterpreterDropNamedCollectionQuery.cpp | 2 +- .../ExternalDataSourceConfiguration.h | 4 +- src/Storages/HeaderCollection.h | 18 --- .../NamedCollectionsHelpers.h | 18 --- .../NamedCollectionsHelpers.cpp | 46 +++++-- src/Storages/NamedCollectionsHelpers.h | 23 ++++ src/Storages/StorageS3.cpp | 63 ++++------ src/Storages/StorageS3.h | 4 +- src/Storages/StorageS3Settings.h | 1 - src/Storages/StorageURL.cpp | 118 +++++++++++------- src/Storages/StorageURL.h | 34 +++-- .../System/StorageSystemNamedCollections.cpp | 2 +- src/TableFunctions/TableFunctionDeltaLake.cpp | 2 +- src/TableFunctions/TableFunctionHudi.cpp | 2 +- src/TableFunctions/TableFunctionS3.cpp | 4 +- src/TableFunctions/TableFunctionURL.cpp | 79 ++++-------- src/TableFunctions/TableFunctionURL.h | 9 +- tests/config/config.d/named_collection.xml | 6 +- .../02244_url_engine_headers_test.reference | 5 +- .../02244_url_engine_headers_test.sql | 9 +- 45 files changed, 459 insertions(+), 312 deletions(-) rename src/{Storages => Common}/NamedCollections/NamedCollectionConfiguration.cpp (86%) rename src/{Storages => Common}/NamedCollections/NamedCollectionConfiguration.h (58%) rename src/{Storages => Common}/NamedCollections/NamedCollectionUtils.cpp (88%) rename src/{Storages => Common}/NamedCollections/NamedCollectionUtils.h (100%) rename src/{Storages => Common}/NamedCollections/NamedCollections.cpp (95%) rename src/{Storages => Common}/NamedCollections/NamedCollections.h (94%) rename src/{Storages => Common}/NamedCollections/NamedCollections_fwd.h (100%) rename src/{Storages => Common}/tests/gtest_named_collections.cpp (61%) create mode 100644 src/IO/HTTPHeaderEntries.h delete mode 100644 src/Storages/HeaderCollection.h delete mode 100644 src/Storages/NamedCollections/NamedCollectionsHelpers.h rename src/Storages/{NamedCollections => }/NamedCollectionsHelpers.cpp (67%) create mode 100644 src/Storages/NamedCollectionsHelpers.h diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 1614fb1a8b4..b415e0d6680 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -37,7 +37,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index db17a6a7bee..0ff05aae770 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -60,7 +60,7 @@ #include #include #include -#include +#include #include #include #include diff --git a/src/Backups/BackupIO_S3.cpp b/src/Backups/BackupIO_S3.cpp index 654651e34be..f8afe309791 100644 --- a/src/Backups/BackupIO_S3.cpp +++ b/src/Backups/BackupIO_S3.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -35,7 +36,7 @@ namespace auto settings = context->getStorageS3Settings().getSettings(s3_uri.uri.toString()); Aws::Auth::AWSCredentials credentials(access_key_id, secret_access_key); - HeaderCollection headers; + HTTPHeaderEntries headers; if (access_key_id.empty()) { credentials = Aws::Auth::AWSCredentials(settings.auth_settings.access_key_id, settings.auth_settings.secret_access_key); diff --git a/src/BridgeHelper/ExternalDictionaryLibraryBridgeHelper.cpp b/src/BridgeHelper/ExternalDictionaryLibraryBridgeHelper.cpp index 4f943e47fa9..aa6eadb4f6e 100644 --- a/src/BridgeHelper/ExternalDictionaryLibraryBridgeHelper.cpp +++ b/src/BridgeHelper/ExternalDictionaryLibraryBridgeHelper.cpp @@ -263,7 +263,7 @@ QueryPipeline ExternalDictionaryLibraryBridgeHelper::loadBase(const Poco::URI & 0, DBMS_DEFAULT_BUFFER_SIZE, getContext()->getReadSettings(), - ReadWriteBufferFromHTTP::HTTPHeaderEntries{}); + HTTPHeaderEntries{}); auto source = FormatFactory::instance().getInput(ExternalDictionaryLibraryBridgeHelper::DEFAULT_FORMAT, *read_buf_ptr, sample_block, getContext(), DEFAULT_BLOCK_SIZE); source->addBuffer(std::move(read_buf_ptr)); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 99c3c0c3fa2..bb3f17b7039 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -106,7 +106,7 @@ if (TARGET ch_contrib::nats_io) endif() add_headers_and_sources(dbms Storages/MeiliSearch) -add_headers_and_sources(dbms Storages/NamedCollections) +add_headers_and_sources(dbms Common/NamedCollections) if (TARGET ch_contrib::amqp_cpp) add_headers_and_sources(dbms Storages/RabbitMQ) diff --git a/src/Storages/NamedCollections/NamedCollectionConfiguration.cpp b/src/Common/NamedCollections/NamedCollectionConfiguration.cpp similarity index 86% rename from src/Storages/NamedCollections/NamedCollectionConfiguration.cpp rename to src/Common/NamedCollections/NamedCollectionConfiguration.cpp index 6875458958b..7fd1635c615 100644 --- a/src/Storages/NamedCollections/NamedCollectionConfiguration.cpp +++ b/src/Common/NamedCollections/NamedCollectionConfiguration.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -145,6 +145,52 @@ ConfigurationPtr createConfiguration(const std::string & root_name, const Settin return config; } +void listKeys( + const Poco::Util::AbstractConfiguration & config, + std::queue enumerate_paths, + std::set> & result, + ssize_t depth) +{ + if (enumerate_paths.empty()) + enumerate_paths.push(""); + + const bool do_finish = depth >= 0 && --depth < 0; + + auto initial_paths = std::move(enumerate_paths); + enumerate_paths = {}; + while (!initial_paths.empty()) + { + auto path = initial_paths.front(); + initial_paths.pop(); + + Poco::Util::AbstractConfiguration::Keys keys; + if (path.empty()) + config.keys(keys); + else + config.keys(path, keys); + + if (keys.empty()) + { + result.insert(path); + } + else if (do_finish) + { + for (const auto & key : keys) + result.emplace(path.empty() ? key : path + '.' + key); + } + else + { + for (const auto & key : keys) + enumerate_paths.emplace(path.empty() ? key : path + '.' + key); + } + } + + if (enumerate_paths.empty()) + return; + + listKeys(config, enumerate_paths, result, depth); +} + template String getConfigValue(const Poco::Util::AbstractConfiguration & config, const std::string & path); template UInt64 getConfigValue(const Poco::Util::AbstractConfiguration & config, diff --git a/src/Storages/NamedCollections/NamedCollectionConfiguration.h b/src/Common/NamedCollections/NamedCollectionConfiguration.h similarity index 58% rename from src/Storages/NamedCollections/NamedCollectionConfiguration.h rename to src/Common/NamedCollections/NamedCollectionConfiguration.h index 7478dcf2d9a..d545ecfb3e6 100644 --- a/src/Storages/NamedCollections/NamedCollectionConfiguration.h +++ b/src/Common/NamedCollections/NamedCollectionConfiguration.h @@ -1,5 +1,7 @@ #pragma once #include +#include +#include namespace DB { @@ -39,6 +41,28 @@ void removeConfigValue( ConfigurationPtr createConfiguration(const std::string & root_name, const SettingsChanges & settings); +/// Enumerate keys paths of the config recursively. +/// E.g. if `enumerate_paths` = {"root.key1"} and config like +/// +/// +/// +/// +/// +/// +/// +/// +/// +/// the `result` will contain: "root.key0", "root.key1.key2" and "root.key1.key3.key4" +/// +/// depth == -1 means to return all keys with full path: "root.key0", "root.key1.key2", "root.key1.key3.key4". +/// depth == 0 means: "root.key0" and "root.key1" +/// depth == 1 means: "root.key0", "root.key1.key2" and "root.key1.key3" +/// and so on. +void listKeys( + const Poco::Util::AbstractConfiguration & config, + std::queue enumerate_paths, + std::set> & result, + ssize_t depth); } } diff --git a/src/Storages/NamedCollections/NamedCollectionUtils.cpp b/src/Common/NamedCollections/NamedCollectionUtils.cpp similarity index 88% rename from src/Storages/NamedCollections/NamedCollectionUtils.cpp rename to src/Common/NamedCollections/NamedCollectionUtils.cpp index c4caa5c95f6..d38a889554b 100644 --- a/src/Storages/NamedCollections/NamedCollectionUtils.cpp +++ b/src/Common/NamedCollections/NamedCollectionUtils.cpp @@ -1,4 +1,4 @@ -#include +#include #include #include #include @@ -13,8 +13,8 @@ #include #include #include -#include -#include +#include +#include namespace fs = std::filesystem; @@ -69,10 +69,10 @@ public: { const auto collection_prefix = getCollectionPrefix(collection_name); std::queue enumerate_input; - std::set enumerate_result; + std::set> enumerate_result; enumerate_input.push(collection_prefix); - collectKeys(config, std::move(enumerate_input), enumerate_result); + NamedCollectionConfiguration::listKeys(config, std::move(enumerate_input), enumerate_result, -1); /// Collection does not have any keys. /// (`enumerate_result` == ). @@ -97,50 +97,6 @@ private: { return fmt::format("{}.{}", NAMED_COLLECTIONS_CONFIG_PREFIX, collection_name); } - - /// Enumerate keys paths of the config recursively. - /// E.g. if `enumerate_paths` = {"root.key1"} and config like - /// - /// - /// - /// - /// - /// - /// - /// - /// - /// the `result` will contain two strings: "root.key1.key2" and "root.key1.key3.key4" - static void collectKeys( - const Poco::Util::AbstractConfiguration & config, - std::queue enumerate_paths, - std::set & result) - { - if (enumerate_paths.empty()) - return; - - auto initial_paths = std::move(enumerate_paths); - enumerate_paths = {}; - while (!initial_paths.empty()) - { - auto path = initial_paths.front(); - initial_paths.pop(); - - Poco::Util::AbstractConfiguration::Keys keys; - config.keys(path, keys); - - if (keys.empty()) - { - result.insert(path); - } - else - { - for (const auto & key : keys) - enumerate_paths.emplace(path + '.' + key); - } - } - - collectKeys(config, enumerate_paths, result); - } }; diff --git a/src/Storages/NamedCollections/NamedCollectionUtils.h b/src/Common/NamedCollections/NamedCollectionUtils.h similarity index 100% rename from src/Storages/NamedCollections/NamedCollectionUtils.h rename to src/Common/NamedCollections/NamedCollectionUtils.h diff --git a/src/Storages/NamedCollections/NamedCollections.cpp b/src/Common/NamedCollections/NamedCollections.cpp similarity index 95% rename from src/Storages/NamedCollections/NamedCollections.cpp rename to src/Common/NamedCollections/NamedCollections.cpp index 03633bbd370..5876a8567f4 100644 --- a/src/Storages/NamedCollections/NamedCollections.cpp +++ b/src/Common/NamedCollections/NamedCollections.cpp @@ -3,8 +3,8 @@ #include #include #include -#include -#include +#include +#include #include #include @@ -229,9 +229,21 @@ public: assert(removed); } - Keys getKeys() const + Keys getKeys(ssize_t depth, const std::string & prefix) const { - return keys; + if (depth == -1) + { + /// Return all keys with full depth. + return keys; + } + + std::queue enumerate_input; + if (!prefix.empty()) + enumerate_input.push(prefix); + + Keys result; + Configuration::listKeys(*config, enumerate_input, result, depth); + return result; } Keys::const_iterator begin() const @@ -379,10 +391,10 @@ MutableNamedCollectionPtr NamedCollection::duplicate() const std::move(impl), collection_name, NamedCollectionUtils::SourceId::NONE, true)); } -NamedCollection::Keys NamedCollection::getKeys() const +NamedCollection::Keys NamedCollection::getKeys(ssize_t depth, const std::string & prefix) const { std::lock_guard lock(mutex); - return pimpl->getKeys(); + return pimpl->getKeys(depth, prefix); } template NamedCollection::const_iterator NamedCollection::begin() const diff --git a/src/Storages/NamedCollections/NamedCollections.h b/src/Common/NamedCollections/NamedCollections.h similarity index 94% rename from src/Storages/NamedCollections/NamedCollections.h rename to src/Common/NamedCollections/NamedCollections.h index 53b7a828a63..5ff9404ed69 100644 --- a/src/Storages/NamedCollections/NamedCollections.h +++ b/src/Common/NamedCollections/NamedCollections.h @@ -1,7 +1,7 @@ #pragma once #include -#include -#include +#include +#include namespace Poco { namespace Util { class AbstractConfiguration; } } @@ -47,7 +47,7 @@ public: MutableNamedCollectionPtr duplicate() const; - Keys getKeys() const; + Keys getKeys(ssize_t depth = -1, const std::string & prefix = "") const; using iterator = typename Keys::iterator; using const_iterator = typename Keys::const_iterator; diff --git a/src/Storages/NamedCollections/NamedCollections_fwd.h b/src/Common/NamedCollections/NamedCollections_fwd.h similarity index 100% rename from src/Storages/NamedCollections/NamedCollections_fwd.h rename to src/Common/NamedCollections/NamedCollections_fwd.h diff --git a/src/Storages/tests/gtest_named_collections.cpp b/src/Common/tests/gtest_named_collections.cpp similarity index 61% rename from src/Storages/tests/gtest_named_collections.cpp rename to src/Common/tests/gtest_named_collections.cpp index d5fe5010991..bd46d2668e4 100644 --- a/src/Storages/tests/gtest_named_collections.cpp +++ b/src/Common/tests/gtest_named_collections.cpp @@ -1,6 +1,6 @@ #include -#include -#include +#include +#include #include #include #include @@ -143,3 +143,82 @@ key2: ASSERT_EQ(collection->get("key2.key2_2.key2_3.key2_5"), 5); } + +TEST(NamedCollections, NestedConfigDuplicateKeys) +{ + std::string xml(R"CONFIG( + + + +
+ key1 + value1 +
+
+ key2 + value2 +
+
+ key3 + value3 +
+ + + +)CONFIG"); + + Poco::XML::DOMParser dom_parser; + Poco::AutoPtr document = dom_parser.parseString(xml); + Poco::AutoPtr config = new Poco::Util::XMLConfiguration(document); + + NamedCollectionUtils::loadFromConfig(*config); + auto collection = NamedCollectionFactory::instance().get("collection"); + + auto keys = collection->getKeys(); + ASSERT_EQ(keys.size(), 6); + + ASSERT_TRUE(keys.contains("headers.header.name")); + ASSERT_TRUE(keys.contains("headers.header[1].name")); + ASSERT_TRUE(keys.contains("headers.header[2].name")); + + ASSERT_TRUE(keys.contains("headers.header.value")); + ASSERT_TRUE(keys.contains("headers.header[1].value")); + ASSERT_TRUE(keys.contains("headers.header[2].value")); + + ASSERT_EQ(collection->get("headers.header.name"), "key1"); + ASSERT_EQ(collection->get("headers.header[1].name"), "key2"); + ASSERT_EQ(collection->get("headers.header[2].name"), "key3"); + + ASSERT_EQ(collection->get("headers.header.value"), "value1"); + ASSERT_EQ(collection->get("headers.header[1].value"), "value2"); + ASSERT_EQ(collection->get("headers.header[2].value"), "value3"); + + keys = collection->getKeys(0); + ASSERT_EQ(keys.size(), 1); + ASSERT_TRUE(keys.contains("headers")); + + keys = collection->getKeys(0, "headers"); + ASSERT_EQ(keys.size(), 3); + + ASSERT_TRUE(keys.contains("headers.header")); + ASSERT_TRUE(keys.contains("headers.header[1]")); + ASSERT_TRUE(keys.contains("headers.header[2]")); + + keys = collection->getKeys(1); + ASSERT_EQ(keys.size(), 3); + + ASSERT_TRUE(keys.contains("headers.header")); + ASSERT_TRUE(keys.contains("headers.header[1]")); + ASSERT_TRUE(keys.contains("headers.header[2]")); + + keys = collection->getKeys(2); + ASSERT_EQ(keys.size(), 6); + + ASSERT_TRUE(keys.contains("headers.header.name")); + ASSERT_TRUE(keys.contains("headers.header[1].name")); + ASSERT_TRUE(keys.contains("headers.header[2].name")); + + ASSERT_TRUE(keys.contains("headers.header.value")); + ASSERT_TRUE(keys.contains("headers.header[1].value")); + ASSERT_TRUE(keys.contains("headers.header[2].value")); +} diff --git a/src/Coordination/KeeperSnapshotManagerS3.cpp b/src/Coordination/KeeperSnapshotManagerS3.cpp index 02451ac36de..982366f8eeb 100644 --- a/src/Coordination/KeeperSnapshotManagerS3.cpp +++ b/src/Coordination/KeeperSnapshotManagerS3.cpp @@ -78,7 +78,7 @@ void KeeperSnapshotManagerS3::updateS3Configuration(const Poco::Util::AbstractCo LOG_INFO(log, "S3 configuration was updated"); auto credentials = Aws::Auth::AWSCredentials(auth_settings.access_key_id, auth_settings.secret_access_key); - HeaderCollection headers = auth_settings.headers; + auto headers = auth_settings.headers; static constexpr size_t s3_max_redirects = 10; static constexpr bool enable_s3_requests_logging = false; diff --git a/src/Dictionaries/HTTPDictionarySource.cpp b/src/Dictionaries/HTTPDictionarySource.cpp index d06e3c4056a..2961c3118cd 100644 --- a/src/Dictionaries/HTTPDictionarySource.cpp +++ b/src/Dictionaries/HTTPDictionarySource.cpp @@ -227,7 +227,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) auto settings_config_prefix = config_prefix + ".http"; Poco::Net::HTTPBasicCredentials credentials; - ReadWriteBufferFromHTTP::HTTPHeaderEntries header_entries; + HTTPHeaderEntries header_entries; String url; String endpoint; String format; @@ -246,7 +246,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) header_entries.reserve(named_collection->configuration.headers.size()); for (const auto & [key, value] : named_collection->configuration.headers) - header_entries.emplace_back(std::make_tuple(key, value)); + header_entries.emplace_back(key, value); } else { @@ -271,7 +271,7 @@ void registerDictionarySourceHTTP(DictionarySourceFactory & factory) { const auto header_key = config.getString(headers_prefix + "." + key + ".name", ""); const auto header_value = config.getString(headers_prefix + "." + key + ".value", ""); - header_entries.emplace_back(std::make_tuple(header_key, header_value)); + header_entries.emplace_back(header_key, header_value); } } diff --git a/src/Dictionaries/HTTPDictionarySource.h b/src/Dictionaries/HTTPDictionarySource.h index 86e3836f2dc..e22aacd89f1 100644 --- a/src/Dictionaries/HTTPDictionarySource.h +++ b/src/Dictionaries/HTTPDictionarySource.h @@ -29,7 +29,7 @@ public: const std::string format; const std::string update_field; const UInt64 update_lag; - const ReadWriteBufferFromHTTP::HTTPHeaderEntries header_entries; + const HTTPHeaderEntries header_entries; }; HTTPDictionarySource( diff --git a/src/Disks/IO/ReadBufferFromWebServer.cpp b/src/Disks/IO/ReadBufferFromWebServer.cpp index 948b595b3c5..41588f7897c 100644 --- a/src/Disks/IO/ReadBufferFromWebServer.cpp +++ b/src/Disks/IO/ReadBufferFromWebServer.cpp @@ -74,7 +74,7 @@ std::unique_ptr ReadBufferFromWebServer::initialize() 0, buf_size, read_settings, - ReadWriteBufferFromHTTP::HTTPHeaderEntries{}, + HTTPHeaderEntries{}, range, &context->getRemoteHostFilter(), /* delay_initialization */true, diff --git a/src/IO/HTTPHeaderEntries.h b/src/IO/HTTPHeaderEntries.h new file mode 100644 index 00000000000..5862f1ead15 --- /dev/null +++ b/src/IO/HTTPHeaderEntries.h @@ -0,0 +1,18 @@ +#pragma once +#include + +namespace DB +{ + +struct HTTPHeaderEntry +{ + std::string name; + std::string value; + + HTTPHeaderEntry(const std::string & name_, const std::string & value_) : name(name_), value(value_) {} + inline bool operator==(const HTTPHeaderEntry & other) const { return name == other.name && value == other.value; } +}; + +using HTTPHeaderEntries = std::vector; + +} diff --git a/src/IO/ReadWriteBufferFromHTTP.h b/src/IO/ReadWriteBufferFromHTTP.h index b60fdee1184..1437b03593f 100644 --- a/src/IO/ReadWriteBufferFromHTTP.h +++ b/src/IO/ReadWriteBufferFromHTTP.h @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -91,9 +92,6 @@ namespace detail class ReadWriteBufferFromHTTPBase : public SeekableReadBuffer, public WithFileName, public WithFileSize { public: - using HTTPHeaderEntry = std::tuple; - using HTTPHeaderEntries = std::vector; - /// HTTP range, including right bound [begin, end]. struct Range { @@ -159,8 +157,8 @@ namespace detail if (out_stream_callback) request.setChunkedTransferEncoding(true); - for (auto & http_header_entry : http_header_entries) - request.set(std::get<0>(http_header_entry), std::get<1>(http_header_entry)); + for (auto & [header, value] : http_header_entries) + request.set(header, value); if (withPartialContent()) { @@ -319,11 +317,11 @@ namespace detail auto iter = std::find_if( http_header_entries.begin(), http_header_entries.end(), - [&user_agent](const HTTPHeaderEntry & entry) { return std::get<0>(entry) == user_agent; }); + [&user_agent](const HTTPHeaderEntry & entry) { return entry.name == user_agent; }); if (iter == http_header_entries.end()) { - http_header_entries.emplace_back(std::make_pair("User-Agent", fmt::format("ClickHouse/{}", VERSION_STRING))); + http_header_entries.emplace_back("User-Agent", fmt::format("ClickHouse/{}", VERSION_STRING)); } if (!delay_initialization) @@ -779,7 +777,7 @@ public: UInt64 max_redirects_ = 0, size_t buffer_size_ = DBMS_DEFAULT_BUFFER_SIZE, ReadSettings settings_ = {}, - ReadWriteBufferFromHTTP::HTTPHeaderEntries http_header_entries_ = {}, + HTTPHeaderEntries http_header_entries_ = {}, const RemoteHostFilter * remote_host_filter_ = nullptr, bool delay_initialization_ = true, bool use_external_buffer_ = false, @@ -851,7 +849,7 @@ private: UInt64 max_redirects; size_t buffer_size; ReadSettings settings; - ReadWriteBufferFromHTTP::HTTPHeaderEntries http_header_entries; + HTTPHeaderEntries http_header_entries; const RemoteHostFilter * remote_host_filter; bool delay_initialization; bool use_external_buffer; diff --git a/src/IO/S3/PocoHTTPClient.h b/src/IO/S3/PocoHTTPClient.h index ed6e1793c2c..762178a9365 100644 --- a/src/IO/S3/PocoHTTPClient.h +++ b/src/IO/S3/PocoHTTPClient.h @@ -11,8 +11,8 @@ #include #include #include +#include #include -#include #include #include @@ -51,7 +51,7 @@ struct PocoHTTPClientConfiguration : public Aws::Client::ClientConfiguration bool for_disk_s3; ThrottlerPtr get_request_throttler; ThrottlerPtr put_request_throttler; - HeaderCollection extra_headers; + HTTPHeaderEntries extra_headers; void updateSchemeAndRegion(); @@ -169,7 +169,7 @@ private: /// NOTE: DELETE and CANCEL requests are not throttled by either put or get throttler ThrottlerPtr put_request_throttler; - const HeaderCollection extra_headers; + const HTTPHeaderEntries extra_headers; }; } diff --git a/src/IO/S3/tests/gtest_aws_s3_client.cpp b/src/IO/S3/tests/gtest_aws_s3_client.cpp index 3b7152c0e7e..da1a43a1dec 100644 --- a/src/IO/S3/tests/gtest_aws_s3_client.cpp +++ b/src/IO/S3/tests/gtest_aws_s3_client.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include "TestPocoHTTPServer.h" @@ -97,7 +98,7 @@ TEST(IOTestAwsS3Client, AppendExtraSSECHeaders) client_configuration.retryStrategy = std::make_shared(); String server_side_encryption_customer_key_base64 = "Kv/gDqdWVGIT4iDqg+btQvV3lc1idlm4WI+MMOyHOAw="; - DB::HeaderCollection headers; + DB::HTTPHeaderEntries headers; bool use_environment_credentials = false; bool use_insecure_imds_request = false; diff --git a/src/IO/S3Common.cpp b/src/IO/S3Common.cpp index b7c3072e4ae..ad45510635b 100644 --- a/src/IO/S3Common.cpp +++ b/src/IO/S3Common.cpp @@ -9,6 +9,7 @@ # include # include +# include # include # include @@ -700,7 +701,7 @@ namespace S3 const String & access_key_id, const String & secret_access_key, const String & server_side_encryption_customer_key_base64, - HeaderCollection headers, + HTTPHeaderEntries headers, bool use_environment_credentials, bool use_insecure_imds_request) { @@ -916,7 +917,7 @@ AuthSettings AuthSettings::loadFromConfig(const std::string & config_elem, const if (config.has(config_elem + ".use_insecure_imds_request")) use_insecure_imds_request = config.getBool(config_elem + ".use_insecure_imds_request"); - HeaderCollection headers; + HTTPHeaderEntries headers; Poco::Util::AbstractConfiguration::Keys subconfig_keys; config.keys(config_elem, subconfig_keys); for (const std::string & subkey : subconfig_keys) @@ -927,7 +928,7 @@ AuthSettings AuthSettings::loadFromConfig(const std::string & config_elem, const auto delimiter = header_str.find(':'); if (delimiter == std::string::npos) throw Exception("Malformed s3 header value", ErrorCodes::INVALID_CONFIG_PARAMETER); - headers.emplace_back(HttpHeader{header_str.substr(0, delimiter), header_str.substr(delimiter + 1, String::npos)}); + headers.emplace_back(header_str.substr(0, delimiter), header_str.substr(delimiter + 1, String::npos)); } } diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 4eb720b3b85..44a1b1a30d8 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include @@ -18,6 +17,7 @@ #include #include +#include namespace Aws::S3 { @@ -80,7 +80,7 @@ public: const String & access_key_id, const String & secret_access_key, const String & server_side_encryption_customer_key_base64, - HeaderCollection headers, + DB::HTTPHeaderEntries headers, bool use_environment_credentials, bool use_insecure_imds_request); @@ -154,7 +154,7 @@ struct AuthSettings std::string region; std::string server_side_encryption_customer_key_base64; - HeaderCollection headers; + DB::HTTPHeaderEntries headers; std::optional use_environment_credentials; std::optional use_insecure_imds_request; diff --git a/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp b/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp index 866d38f3aa5..040a8714983 100644 --- a/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp +++ b/src/Interpreters/InterpreterAlterNamedCollectionQuery.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp b/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp index ccbbac71279..8a1a8d9dde6 100644 --- a/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp +++ b/src/Interpreters/InterpreterCreateNamedCollectionQuery.cpp @@ -4,7 +4,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp b/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp index 2205d6cff88..064a13012a6 100644 --- a/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp +++ b/src/Interpreters/InterpreterDropNamedCollectionQuery.cpp @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index a8c27e3d1d4..2045f1d6d9f 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -3,7 +3,7 @@ #include #include #include -#include +#include namespace DB @@ -109,7 +109,7 @@ struct URLBasedDataSourceConfiguration String user; String password; - HeaderCollection headers; + HTTPHeaderEntries headers; String http_method; void set(const URLBasedDataSourceConfiguration & conf); diff --git a/src/Storages/HeaderCollection.h b/src/Storages/HeaderCollection.h deleted file mode 100644 index 233553304a1..00000000000 --- a/src/Storages/HeaderCollection.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once -#include - -namespace DB -{ - -struct HttpHeader -{ - std::string name; - std::string value; - - HttpHeader(const std::string & name_, const std::string & value_) : name(name_), value(value_) {} - inline bool operator==(const HttpHeader & other) const { return name == other.name && value == other.value; } -}; - -using HeaderCollection = std::vector; - -} diff --git a/src/Storages/NamedCollections/NamedCollectionsHelpers.h b/src/Storages/NamedCollections/NamedCollectionsHelpers.h deleted file mode 100644 index 39baafa9039..00000000000 --- a/src/Storages/NamedCollections/NamedCollectionsHelpers.h +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once -#include -#include -#include -#include - - -namespace DB -{ - -NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts); - -void validateNamedCollection( - const NamedCollection & collection, - const std::unordered_set & required_keys, - const std::unordered_set & optional_keys); - -} diff --git a/src/Storages/NamedCollections/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp similarity index 67% rename from src/Storages/NamedCollections/NamedCollectionsHelpers.cpp rename to src/Storages/NamedCollectionsHelpers.cpp index cceabdfd7bf..b2935bb0fce 100644 --- a/src/Storages/NamedCollections/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -1,5 +1,5 @@ #include "NamedCollectionsHelpers.h" -#include +#include #include #include #include @@ -83,12 +83,30 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) void validateNamedCollection( const NamedCollection & collection, const std::unordered_set & required_keys, - const std::unordered_set & optional_keys) + const std::unordered_set & optional_keys, + const std::vector & optional_regex_keys) { const auto & keys = collection.getKeys(); + auto required_keys_copy = required_keys; + for (const auto & key : keys) { - if (!required_keys.contains(key) && !optional_keys.contains(key)) + auto it = required_keys_copy.find(key); + if (it != required_keys_copy.end()) + { + required_keys_copy.erase(it); + continue; + } + + if (optional_keys.contains(key)) + continue; + + auto match = std::find_if( + optional_regex_keys.begin(), optional_regex_keys.end(), + [&](const std::regex & regex) { return std::regex_search(key, regex); }) + != optional_regex_keys.end(); + + if (!match) { throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -97,16 +115,22 @@ void validateNamedCollection( } } - for (const auto & key : required_keys) + if (!required_keys_copy.empty()) { - if (!keys.contains(key)) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Key `{}` is required, but not specified. Required keys: {}, optional keys: {}", - key, fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); - } + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Required keys ({}) are not specified. All required keys: {}, optional keys: {}", + fmt::join(required_keys_copy, ", "), fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); } } +HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collection) +{ + HTTPHeaderEntries headers; + auto keys = collection.getKeys(0, "headers"); + for (const auto & key : keys) + headers.emplace_back(collection.get(key + ".name"), collection.get(key + ".value")); + return headers; +} + } diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h new file mode 100644 index 00000000000..029f9f41ebf --- /dev/null +++ b/src/Storages/NamedCollectionsHelpers.h @@ -0,0 +1,23 @@ +#pragma once +#include +#include +#include +#include +#include +#include + + +namespace DB +{ + +NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts); + +void validateNamedCollection( + const NamedCollection & collection, + const std::unordered_set & required_keys, + const std::unordered_set & optional_keys, + const std::vector & optional_regex_keys = {}); + +HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collection); + +} diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 61437f07c65..2646d577770 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -28,8 +28,8 @@ #include #include #include -#include -#include +#include +#include #include #include @@ -1200,48 +1200,27 @@ void StorageS3::updateS3Configuration(ContextPtr ctx, StorageS3::S3Configuration void StorageS3::processNamedCollectionResult(StorageS3Configuration & configuration, const NamedCollection & collection) { validateNamedCollection(collection, required_configuration_keys, optional_configuration_keys); - std::string filename; - for (const auto & key : collection) - { - if (key == "url") - configuration.url = collection.get(key); - else if (key == "access_key_id") - configuration.auth_settings.access_key_id = collection.get(key); - else if (key == "secret_access_key") - configuration.auth_settings.secret_access_key = collection.get(key); - else if (key == "filename") - filename = collection.get(key); - else if (key == "format") - configuration.format = collection.get(key); - else if (key == "compression" || key == "compression_method") - configuration.compression_method = collection.get(key); - else if (key == "structure") - configuration.structure = collection.get(key); - else if (key == "use_environment_credentials") - configuration.auth_settings.use_environment_credentials = collection.get(key); - else if (key == "max_single_read_retries") - configuration.request_settings.max_single_read_retries = collection.get(key); - else if (key == "min_upload_part_size") - configuration.request_settings.min_upload_part_size = collection.get(key); - else if (key == "upload_part_size_multiply_factor") - configuration.request_settings.upload_part_size_multiply_factor = collection.get(key); - else if (key == "upload_part_size_multiply_parts_count_threshold") - configuration.request_settings.upload_part_size_multiply_parts_count_threshold = collection.get(key); - else if (key == "max_single_part_upload_size") - configuration.request_settings.max_single_part_upload_size = collection.get(key); - else if (key == "max_connections") - configuration.request_settings.max_connections = collection.get(key); - else - throw Exception( - ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Unknown configuration key `{}` for StorageS3, " - "expected: url, [access_key_id, secret_access_key], " - "name of used format and [compression_method].", - key); - } + configuration.url = collection.get("url"); + + auto filename = collection.getOrDefault("filename", ""); if (!filename.empty()) configuration.url = std::filesystem::path(configuration.url) / filename; + + configuration.auth_settings.access_key_id = collection.getOrDefault("access_key_id", ""); + configuration.auth_settings.secret_access_key = collection.getOrDefault("secret_access_key", ""); + configuration.auth_settings.use_environment_credentials = collection.getOrDefault("use_environment_credentials", 0); + + configuration.format = collection.getOrDefault("format", "auto"); + configuration.compression_method = collection.getOrDefault("compression_method", collection.getOrDefault("compression", "auto")); + configuration.structure = collection.getOrDefault("structure", "auto"); + + configuration.request_settings.max_single_read_retries = collection.getOrDefault("max_single_read_retries", 0); + configuration.request_settings.min_upload_part_size = collection.getOrDefault("min_upload_part_size", 0); + configuration.request_settings.upload_part_size_multiply_factor = collection.getOrDefault("upload_part_size_multiply_factor", 0); + configuration.request_settings.upload_part_size_multiply_parts_count_threshold = collection.getOrDefault("upload_part_size_multiply_parts_count_threshold", 0); + configuration.request_settings.max_single_part_upload_size = collection.getOrDefault("max_single_part_upload_size", 0); + configuration.request_settings.max_connections = collection.getOrDefault("max_connections", 0); } StorageS3Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPtr local_context) @@ -1267,7 +1246,7 @@ StorageS3Configuration StorageS3::getConfiguration(ASTs & engine_args, ContextPt "Storage S3 requires 1 to 5 arguments: url, [access_key_id, secret_access_key], name of used format and [compression_method].", ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH); - auto header_it = StorageURL::collectHeaders(engine_args, configuration, local_context); + auto header_it = StorageURL::collectHeaders(engine_args, configuration.headers, local_context); if (header_it != engine_args.end()) engine_args.erase(header_it); diff --git a/src/Storages/StorageS3.h b/src/Storages/StorageS3.h index 5bf714d5602..e1a52e3f023 100644 --- a/src/Storages/StorageS3.h +++ b/src/Storages/StorageS3.h @@ -233,13 +233,13 @@ public: bool static_configuration = true; /// Headers from ast is a part of static configuration. - HeaderCollection headers_from_ast; + HTTPHeaderEntries headers_from_ast; S3Configuration( const String & url_, const S3::AuthSettings & auth_settings_, const S3Settings::RequestSettings & request_settings_, - const HeaderCollection & headers_from_ast_) + const HTTPHeaderEntries & headers_from_ast_) : uri(S3::URI(url_)) , auth_settings(auth_settings_) , request_settings(request_settings_) diff --git a/src/Storages/StorageS3Settings.h b/src/Storages/StorageS3Settings.h index bf04dbe3a61..7c809a273fb 100644 --- a/src/Storages/StorageS3Settings.h +++ b/src/Storages/StorageS3Settings.h @@ -8,7 +8,6 @@ #include #include #include -#include #include diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 0f01dc4288c..6b875c2d138 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -27,6 +28,7 @@ #include #include +#include #include #include @@ -34,6 +36,7 @@ #include #include #include +#include namespace DB @@ -50,6 +53,27 @@ static constexpr auto bad_arguments_error_message = "Storage URL requires 1-4 ar "url, name of used format (taken from file extension by default), " "optional compression method, optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; +static const std::unordered_set required_configuration_keys = { + "url", +}; + +static const std::unordered_set optional_configuration_keys = { + "format", + "compression", + "compression_method", + "structure", + "filename", + "headers.header.name", + "headers.header.value", +}; + +/// Headers in config file will have structure "headers.header.name" and "headers.header.value". +/// But Poco::AbstractConfiguration converts them into "header", "header[1]", "header[2]". +static const std::vector optional_regex_keys = { + std::regex("headers.header\\[[\\d]*\\].name"), + std::regex("headers.header\\[[\\d]*\\].value"), +}; + static bool urlWithGlobs(const String & uri) { return (uri.find('{') != std::string::npos && uri.find('}') != std::string::npos) || uri.find('|') != std::string::npos; @@ -66,7 +90,7 @@ IStorageURLBase::IStorageURLBase( const ConstraintsDescription & constraints_, const String & comment, const String & compression_method_, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_, + const HTTPHeaderEntries & headers_, const String & http_method_, ASTPtr partition_by_) : IStorage(table_id_) @@ -97,9 +121,9 @@ IStorageURLBase::IStorageURLBase( namespace { - ReadWriteBufferFromHTTP::HTTPHeaderEntries getHeaders(const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_) + HTTPHeaderEntries getHeaders(const HTTPHeaderEntries & headers_) { - ReadWriteBufferFromHTTP::HTTPHeaderEntries headers(headers_.begin(), headers_.end()); + HTTPHeaderEntries headers(headers_.begin(), headers_.end()); // Propagate OpenTelemetry trace context, if any, downstream. const auto ¤t_trace_context = OpenTelemetry::CurrentContext(); @@ -165,7 +189,7 @@ namespace const ConnectionTimeouts & timeouts, CompressionMethod compression_method, size_t download_threads, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_ = {}, + const HTTPHeaderEntries & headers_ = {}, const URIParams & params = {}, bool glob_url = false) : ISource(sample_block), name(std::move(name_)), uri_info(uri_info_) @@ -246,7 +270,7 @@ namespace const ConnectionTimeouts & timeouts, CompressionMethod compression_method, Poco::Net::HTTPBasicCredentials & credentials, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + const HTTPHeaderEntries & headers, bool glob_url, bool delay_initialization, size_t download_threads) @@ -567,7 +591,7 @@ ColumnsDescription IStorageURLBase::getTableStructureFromData( const String & format, const String & uri, CompressionMethod compression_method, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + const HTTPHeaderEntries & headers, const std::optional & format_settings, ContextPtr context) { @@ -816,7 +840,7 @@ SchemaCache & IStorageURLBase::getSchemaCache(const ContextPtr & context) std::optional IStorageURLBase::tryGetColumnsFromCache( const Strings & urls, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + const HTTPHeaderEntries & headers, const Poco::Net::HTTPBasicCredentials & credentials, const String & format_name, const std::optional & format_settings, @@ -859,7 +883,7 @@ void IStorageURLBase::addColumnsToCache( std::optional IStorageURLBase::getLastModificationTime( const String & url, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + const HTTPHeaderEntries & headers, const Poco::Net::HTTPBasicCredentials & credentials, const ContextPtr & context) { @@ -901,7 +925,7 @@ StorageURL::StorageURL( const String & comment, ContextPtr context_, const String & compression_method_, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_, + const HTTPHeaderEntries & headers_, const String & http_method_, ASTPtr partition_by_) : IStorageURLBase( @@ -978,15 +1002,18 @@ FormatSettings StorageURL::getFormatSettingsFromArgs(const StorageFactory::Argum } ASTs::iterator StorageURL::collectHeaders( - ASTs & url_function_args, URLBasedDataSourceConfiguration & configuration, ContextPtr context) + ASTs & url_function_args, HTTPHeaderEntries & header_entries, ContextPtr context) { ASTs::iterator headers_it = url_function_args.end(); for (auto arg_it = url_function_args.begin(); arg_it != url_function_args.end(); ++arg_it) { const auto * headers_ast_function = (*arg_it)->as(); - if (headers_ast_function && headers_ast_function->name == "headers") + if (headers_ast_function) { + if (headers_ast_function->name != "headers") + continue; + if (headers_it != url_function_args.end()) throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -1021,7 +1048,7 @@ ASTs::iterator StorageURL::collectHeaders( if (arg_value.getType() != Field::Types::Which::String) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected string as header value"); - configuration.headers.emplace_back(arg_name, arg_value.safeGet()); + header_entries.emplace_back(arg_name, arg_value.safeGet()); } headers_it = arg_it; @@ -1035,40 +1062,41 @@ ASTs::iterator StorageURL::collectHeaders( return headers_it; } -URLBasedDataSourceConfiguration StorageURL::getConfiguration(ASTs & args, ContextPtr local_context) +void StorageURL::processNamedCollectionResult(Configuration & configuration, const NamedCollection & collection) { - URLBasedDataSourceConfiguration configuration; + validateNamedCollection(collection, required_configuration_keys, optional_configuration_keys, optional_regex_keys); - if (auto named_collection = getURLBasedDataSourceConfiguration(args, local_context)) + configuration.url = collection.get("url"); + configuration.headers = getHeadersFromNamedCollection(collection); + + configuration.http_method = collection.getOrDefault("http_method", ""); + if (!configuration.http_method.empty() && configuration.http_method != Poco::Net::HTTPRequest::HTTP_POST + && configuration.http_method != Poco::Net::HTTPRequest::HTTP_PUT) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Http method can be POST or PUT (current: {}). For insert default is POST, for select GET", + configuration.http_method); + + configuration.format = collection.getOrDefault("format", "auto"); + configuration.compression_method = collection.getOrDefault("compression_method", collection.getOrDefault("compression", "auto")); + configuration.structure = collection.getOrDefault("structure", "auto"); +} + +StorageURL::Configuration StorageURL::getConfiguration(ASTs & args, ContextPtr local_context) +{ + StorageURL::Configuration configuration; + + if (auto named_collection = tryGetNamedCollectionWithOverrides(args)) { - auto [common_configuration, storage_specific_args] = named_collection.value(); - configuration.set(common_configuration); - - if (!configuration.http_method.empty() && configuration.http_method != Poco::Net::HTTPRequest::HTTP_POST - && configuration.http_method != Poco::Net::HTTPRequest::HTTP_PUT) - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Http method can be POST or PUT (current: {}). For insert default is POST, for select GET", - configuration.http_method); - - if (!storage_specific_args.empty()) - { - String illegal_args; - for (const auto & arg : storage_specific_args) - { - if (!illegal_args.empty()) - illegal_args += ", "; - illegal_args += arg.first; - } - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown argument `{}` for storage URL", illegal_args); - } + StorageURL::processNamedCollectionResult(configuration, *named_collection); + collectHeaders(args, configuration.headers, local_context); } else { if (args.empty() || args.size() > 3) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, bad_arguments_error_message); - auto header_it = collectHeaders(args, configuration, local_context); + auto header_it = collectHeaders(args, configuration.headers, local_context); if (header_it != args.end()) args.erase(header_it); @@ -1082,6 +1110,12 @@ URLBasedDataSourceConfiguration StorageURL::getConfiguration(ASTs & args, Contex if (configuration.format == "auto") configuration.format = FormatFactory::instance().getFormatFromFileName(Poco::URI(configuration.url).getPath(), true); + for (const auto & [header, value] : configuration.headers) + { + if (header == "Range") + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Range headers are not allowed"); + } + return configuration; } @@ -1096,14 +1130,6 @@ void registerStorageURL(StorageFactory & factory) auto configuration = StorageURL::getConfiguration(engine_args, args.getLocalContext()); auto format_settings = StorageURL::getFormatSettingsFromArgs(args); - ReadWriteBufferFromHTTP::HTTPHeaderEntries headers; - for (const auto & [header, value] : configuration.headers) - { - if (header == "Range") - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Range headers are not allowed"); - headers.emplace_back(header, value); - } - ASTPtr partition_by; if (args.storage_def->partition_by) partition_by = args.storage_def->partition_by->clone(); @@ -1118,7 +1144,7 @@ void registerStorageURL(StorageFactory & factory) args.comment, args.getContext(), configuration.compression_method, - headers, + configuration.headers, configuration.http_method, partition_by); }, diff --git a/src/Storages/StorageURL.h b/src/Storages/StorageURL.h index bf8858b8b66..6666e2a716e 100644 --- a/src/Storages/StorageURL.h +++ b/src/Storages/StorageURL.h @@ -1,11 +1,12 @@ #pragma once -#include #include #include #include #include #include +#include +#include #include #include #include @@ -18,6 +19,7 @@ class IOutputFormat; using OutputFormatPtr = std::shared_ptr; struct ConnectionTimeouts; +class NamedCollection; /** * This class represents table engine for external urls. @@ -45,7 +47,7 @@ public: const String & format, const String & uri, CompressionMethod compression_method, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + const HTTPHeaderEntries & headers, const std::optional & format_settings, ContextPtr context); @@ -62,7 +64,7 @@ protected: const ConstraintsDescription & constraints_, const String & comment, const String & compression_method_, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_ = {}, + const HTTPHeaderEntries & headers_ = {}, const String & method_ = "", ASTPtr partition_by = nullptr); @@ -74,7 +76,7 @@ protected: // For `url` table function, we use settings from current query context. // In this case, format_settings is not set. std::optional format_settings; - ReadWriteBufferFromHTTP::HTTPHeaderEntries headers; + HTTPHeaderEntries headers; String http_method; /// For insert can choose Put instead of default Post. ASTPtr partition_by; @@ -103,7 +105,7 @@ private: static std::optional tryGetColumnsFromCache( const Strings & urls, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + const HTTPHeaderEntries & headers, const Poco::Net::HTTPBasicCredentials & credentials, const String & format_name, const std::optional & format_settings, @@ -118,7 +120,7 @@ private: static std::optional getLastModificationTime( const String & url, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers, + const HTTPHeaderEntries & headers, const Poco::Net::HTTPBasicCredentials & credentials, const ContextPtr & context); }; @@ -163,7 +165,7 @@ public: const String & comment, ContextPtr context_, const String & compression_method_, - const ReadWriteBufferFromHTTP::HTTPHeaderEntries & headers_ = {}, + const HTTPHeaderEntries & headers_ = {}, const String & method_ = "", ASTPtr partition_by_ = nullptr); @@ -179,9 +181,23 @@ public: static FormatSettings getFormatSettingsFromArgs(const StorageFactory::Arguments & args); - static URLBasedDataSourceConfiguration getConfiguration(ASTs & args, ContextPtr context); + struct Configuration + { + std::string url; + std::string http_method; - static ASTs::iterator collectHeaders(ASTs & url_function_args, URLBasedDataSourceConfiguration & configuration, ContextPtr context); + std::string format = "auto"; + std::string compression_method = "auto"; + std::string structure = "auto"; + + HTTPHeaderEntries headers; + }; + + static Configuration getConfiguration(ASTs & args, ContextPtr context); + + static ASTs::iterator collectHeaders(ASTs & url_function_args, HTTPHeaderEntries & header_entries, ContextPtr context); + + static void processNamedCollectionResult(Configuration & configuration, const NamedCollection & collection); }; diff --git a/src/Storages/System/StorageSystemNamedCollections.cpp b/src/Storages/System/StorageSystemNamedCollections.cpp index 16c259796e6..bc1e3a45e6b 100644 --- a/src/Storages/System/StorageSystemNamedCollections.cpp +++ b/src/Storages/System/StorageSystemNamedCollections.cpp @@ -8,7 +8,7 @@ #include #include #include -#include +#include namespace DB diff --git a/src/TableFunctions/TableFunctionDeltaLake.cpp b/src/TableFunctions/TableFunctionDeltaLake.cpp index f831d4ae609..54a2b66d73f 100644 --- a/src/TableFunctions/TableFunctionDeltaLake.cpp +++ b/src/TableFunctions/TableFunctionDeltaLake.cpp @@ -33,7 +33,7 @@ void TableFunctionDeltaLake::parseArgumentsImpl( if (args.empty() || args.size() > 6) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message); - auto header_it = StorageURL::collectHeaders(args, base_configuration, context); + auto header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); if (header_it != args.end()) args.erase(header_it); diff --git a/src/TableFunctions/TableFunctionHudi.cpp b/src/TableFunctions/TableFunctionHudi.cpp index f39f3b515ec..79b16398023 100644 --- a/src/TableFunctions/TableFunctionHudi.cpp +++ b/src/TableFunctions/TableFunctionHudi.cpp @@ -33,7 +33,7 @@ void TableFunctionHudi::parseArgumentsImpl( if (args.empty() || args.size() > 6) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message); - auto header_it = StorageURL::collectHeaders(args, base_configuration, context); + auto header_it = StorageURL::collectHeaders(args, base_configuration.headers, context); if (header_it != args.end()) args.erase(header_it); diff --git a/src/TableFunctions/TableFunctionS3.cpp b/src/TableFunctions/TableFunctionS3.cpp index 23822486c29..26638c22dc7 100644 --- a/src/TableFunctions/TableFunctionS3.cpp +++ b/src/TableFunctions/TableFunctionS3.cpp @@ -13,7 +13,7 @@ #include #include #include -#include +#include #include #include "registerTableFunctions.h" #include @@ -40,7 +40,7 @@ void TableFunctionS3::parseArgumentsImpl(const String & error_message, ASTs & ar if (args.empty() || args.size() > 6) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, error_message); - auto header_it = StorageURL::collectHeaders(args, s3_configuration, context); + auto header_it = StorageURL::collectHeaders(args, s3_configuration.headers, context); if (header_it != args.end()) args.erase(header_it); diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index 888f3e7b93d..232471e0b1e 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -6,8 +6,8 @@ #include #include #include -#include #include +#include #include #include #include @@ -16,67 +16,48 @@ namespace DB { +static const String bad_arguments_error_message = "Table function URL can have the following arguments: " + "url, name of used format (taken from file extension by default), " + "optional table structure, optional compression method, " + "optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; namespace ErrorCodes { extern const int BAD_ARGUMENTS; } -void TableFunctionURL::parseArguments(const ASTPtr & ast_function, ContextPtr context) +void TableFunctionURL::parseArguments(const ASTPtr & ast, ContextPtr context) { - const auto & func_args = ast_function->as(); - if (!func_args.arguments) - throw Exception("Table function 'URL' must have arguments.", ErrorCodes::BAD_ARGUMENTS); + const auto & ast_function = assert_cast(ast.get()); - if (auto with_named_collection = getURLBasedDataSourceConfiguration(func_args.arguments->children, context)) + auto & args = ast_function->children; + if (args.empty()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, bad_arguments_error_message); + + auto & url_function_args = assert_cast(args[0].get())->children; + + if (auto named_collection = tryGetNamedCollectionWithOverrides(url_function_args)) { - auto [common_configuration, storage_specific_args] = with_named_collection.value(); - configuration.set(common_configuration); - - if (!configuration.http_method.empty() - && configuration.http_method != Poco::Net::HTTPRequest::HTTP_POST - && configuration.http_method != Poco::Net::HTTPRequest::HTTP_PUT) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Method can be POST or PUT (current: {}). For insert default is POST, for select GET", - configuration.http_method); - - if (!storage_specific_args.empty()) - { - String illegal_args; - for (const auto & arg : storage_specific_args) - { - if (!illegal_args.empty()) - illegal_args += ", "; - illegal_args += arg.first; - } - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown argument `{}` for table function URL", illegal_args); - } + StorageURL::processNamedCollectionResult(configuration, *named_collection); filename = configuration.url; + structure = configuration.structure; + compression_method = configuration.compression_method; + format = configuration.format; if (format == "auto") format = FormatFactory::instance().getFormatFromFileName(Poco::URI(filename).getPath(), true); - structure = configuration.structure; - compression_method = configuration.compression_method; + + StorageURL::collectHeaders(url_function_args, configuration.headers, context); } else { - String bad_arguments_error_message = "Table function URL can have the following arguments: " - "url, name of used format (taken from file extension by default), " - "optional table structure, optional compression method, optional headers (specified as `headers('name'='value', 'name2'='value2')`)"; - - auto & args = ast_function->children; - if (args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, bad_arguments_error_message); - - auto * url_function_args_expr = assert_cast(args[0].get()); - auto & url_function_args = url_function_args_expr->children; - auto headers_it = StorageURL::collectHeaders(url_function_args, configuration, context); + auto headers_it = StorageURL::collectHeaders(url_function_args, configuration.headers, context); /// ITableFunctionFileLike cannot parse headers argument, so remove it. if (headers_it != url_function_args.end()) url_function_args.erase(headers_it); - ITableFunctionFileLike::parseArguments(ast_function, context); + ITableFunctionFileLike::parseArguments(ast, context); } } @@ -94,22 +75,10 @@ StoragePtr TableFunctionURL::getStorage( String{}, global_context, compression_method_, - getHeaders(), + configuration.headers, configuration.http_method); } -ReadWriteBufferFromHTTP::HTTPHeaderEntries TableFunctionURL::getHeaders() const -{ - ReadWriteBufferFromHTTP::HTTPHeaderEntries headers; - for (const auto & [header, value] : configuration.headers) - { - if (header == "Range") - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Range headers are not allowed"); - headers.emplace_back(header, value); - } - return headers; -} - ColumnsDescription TableFunctionURL::getActualTableStructure(ContextPtr context) const { if (structure == "auto") @@ -118,7 +87,7 @@ ColumnsDescription TableFunctionURL::getActualTableStructure(ContextPtr context) return StorageURL::getTableStructureFromData(format, filename, chooseCompressionMethod(Poco::URI(filename).getPath(), compression_method), - getHeaders(), + configuration.headers, std::nullopt, context); } diff --git a/src/TableFunctions/TableFunctionURL.h b/src/TableFunctions/TableFunctionURL.h index 9fad42c8d2d..a670bdc0682 100644 --- a/src/TableFunctions/TableFunctionURL.h +++ b/src/TableFunctions/TableFunctionURL.h @@ -1,7 +1,7 @@ #pragma once #include -#include +#include #include @@ -24,19 +24,18 @@ public: ColumnsDescription getActualTableStructure(ContextPtr context) const override; protected: - void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; + void parseArguments(const ASTPtr & ast, ContextPtr context) override; private: StoragePtr getStorage( const String & source, const String & format_, const ColumnsDescription & columns, ContextPtr global_context, const std::string & table_name, const String & compression_method_) const override; + const char * getStorageTypeName() const override { return "URL"; } String getFormatFromFirstArgument() override; - ReadWriteBufferFromHTTP::HTTPHeaderEntries getHeaders() const; - - URLBasedDataSourceConfiguration configuration; + StorageURL::Configuration configuration; }; } diff --git a/tests/config/config.d/named_collection.xml b/tests/config/config.d/named_collection.xml index f7b731061a8..2e49c0c596f 100644 --- a/tests/config/config.d/named_collection.xml +++ b/tests/config/config.d/named_collection.xml @@ -18,7 +18,11 @@
X-ClickHouse-Format - JSONEachRow + Vertical +
+
+ X-ClickHouse-Database + non_existing_database
diff --git a/tests/queries/0_stateless/02244_url_engine_headers_test.reference b/tests/queries/0_stateless/02244_url_engine_headers_test.reference index 1ffacdcdbf6..7a71ce66b3b 100644 --- a/tests/queries/0_stateless/02244_url_engine_headers_test.reference +++ b/tests/queries/0_stateless/02244_url_engine_headers_test.reference @@ -1,3 +1,6 @@ -{"12":12}\n +Row 1:\n──────\n12: 12\n +{"12":12}\n +12\n +12\n {"12":12}\n {"12":12}\n diff --git a/tests/queries/0_stateless/02244_url_engine_headers_test.sql b/tests/queries/0_stateless/02244_url_engine_headers_test.sql index 2a2557f629b..6df01055289 100644 --- a/tests/queries/0_stateless/02244_url_engine_headers_test.sql +++ b/tests/queries/0_stateless/02244_url_engine_headers_test.sql @@ -1,5 +1,10 @@ -select * from url(url_with_headers, url='http://127.0.0.1:8123?query=select+12', format='RawBLOB'); -select * from url('http://127.0.0.1:8123?query=select+12', 'RawBLOB', headers('X-ClickHouse-Format'='JSONEachRow')); +select * from url(url_with_headers, url='http://127.0.0.1:8123?query=select+12', format='RawBLOB'); -- { serverError 86 } +select * from url(url_with_headers, url='http://127.0.0.1:8123?query=select+12', format='RawBLOB', headers('X-ClickHouse-Database'='default')); +select * from url(url_with_headers, url='http://127.0.0.1:8123?query=select+12', format='RawBLOB', headers('X-ClickHouse-Database'='default', 'X-ClickHouse-Format'='JSONEachRow')); +select * from url(url_with_headers, url='http://127.0.0.1:8123?query=select+12', format='RawBLOB', headers('X-ClickHouse-Database'='kek')); -- { serverError 86 } +select * from url('http://127.0.0.1:8123?query=select+12', 'RawBLOB'); +select * from url('http://127.0.0.1:8123?query=select+12', 'RawBLOB', headers('X-ClickHouse-Database'='default')); +select * from url('http://127.0.0.1:8123?query=select+12', 'RawBLOB', headers('X-ClickHouse-Database'='default', 'X-ClickHouse-Format'='JSONEachRow')); select * from url('http://127.0.0.1:8123?query=select+12', 'RawBLOB', headers('X-ClickHouse-Format'='JSONEachRow', 'X-ClickHouse-Database'='kek')); -- { serverError 86 } select * from url('http://127.0.0.1:8123?query=select+12', 'RawBLOB', headers('X-ClickHouse-Format'='JSONEachRow', 'X-ClickHouse-Database'=1)); -- { serverError 36 } drop table if exists url; From 0c63ce9731cbd660cb2b664da880a938188c2fc6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 17 Dec 2022 00:34:29 +0100 Subject: [PATCH 06/88] Remove some more old code --- .../ExternalDataSourceConfiguration.cpp | 104 ------------------ .../ExternalDataSourceConfiguration.h | 2 - src/Storages/StorageExternalDistributed.cpp | 22 +--- src/Storages/StorageURL.cpp | 1 + 4 files changed, 6 insertions(+), 123 deletions(-) diff --git a/src/Storages/ExternalDataSourceConfiguration.cpp b/src/Storages/ExternalDataSourceConfiguration.cpp index 7c537d503b0..c13c3c4f1ef 100644 --- a/src/Storages/ExternalDataSourceConfiguration.cpp +++ b/src/Storages/ExternalDataSourceConfiguration.cpp @@ -399,110 +399,6 @@ void URLBasedDataSourceConfiguration::set(const URLBasedDataSourceConfiguration headers = conf.headers; } - -std::optional getURLBasedDataSourceConfiguration(const ASTs & args, ContextPtr context) -{ - if (args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "External data source must have arguments"); - - URLBasedDataSourceConfiguration configuration; - StorageSpecificArgs non_common_args; - - if (const auto * collection = typeid_cast(args[0].get())) - { - const auto & config = context->getConfigRef(); - auto config_prefix = fmt::format("named_collections.{}", collection->name()); - - if (!config.has(config_prefix)) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "There is no collection named `{}` in config", collection->name()); - - Poco::Util::AbstractConfiguration::Keys keys; - config.keys(config_prefix, keys); - for (const auto & key : keys) - { - if (key == "url") - { - configuration.url = config.getString(config_prefix + ".url", ""); - } - else if (key == "method") - { - configuration.http_method = config.getString(config_prefix + ".method", ""); - } - else if (key == "format") - { - configuration.format = config.getString(config_prefix + ".format", ""); - } - else if (key == "structure") - { - configuration.structure = config.getString(config_prefix + ".structure", ""); - } - else if (key == "compression_method") - { - configuration.compression_method = config.getString(config_prefix + ".compression_method", ""); - } - else if (key == "headers") - { - Poco::Util::AbstractConfiguration::Keys header_keys; - config.keys(config_prefix + ".headers", header_keys); - for (const auto & header : header_keys) - { - const auto header_prefix = config_prefix + ".headers." + header; - configuration.headers.emplace_back( - config.getString(header_prefix + ".name"), - config.getString(header_prefix + ".value")); - } - } - else - { - auto value = config.getString(config_prefix + '.' + key); - non_common_args.emplace_back(std::make_pair(key, std::make_shared(value))); - } - } - - /// Check key-value arguments. - for (size_t i = 1; i < args.size(); ++i) - { - if (const auto * ast_function = typeid_cast(args[i].get())) - { - const auto * args_expr = assert_cast(ast_function->arguments.get()); - auto function_args = args_expr->children; - if (function_args.size() != 2) - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - - auto arg_name = function_args[0]->as()->name(); - auto arg_value_ast = evaluateConstantExpressionOrIdentifierAsLiteral(function_args[1], context); - auto arg_value = arg_value_ast->as()->value; - - if (arg_name == "url") - configuration.url = arg_value.safeGet(); - else if (arg_name == "method") - configuration.http_method = arg_value.safeGet(); - else if (arg_name == "format") - configuration.format = arg_value.safeGet(); - else if (arg_name == "compression_method") - configuration.compression_method = arg_value.safeGet(); - else if (arg_name == "structure") - configuration.structure = arg_value.safeGet(); - else - non_common_args.emplace_back(std::make_pair(arg_name, arg_value_ast)); - } - else - { - throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value defined argument"); - } - } - - if (configuration.url.empty() || configuration.format.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Storage requires {}", configuration.url.empty() ? "url" : "format"); - - URLBasedDataSourceConfig source_config{ .configuration = configuration, .specific_args = non_common_args }; - return source_config; - } - return std::nullopt; -} - - template bool getExternalDataSourceConfiguration(const ASTs & args, BaseSettings & settings, ContextPtr context) { diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index 2045f1d6d9f..ed6d7b3447f 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -133,8 +133,6 @@ struct URLBasedDataSourceConfig StorageSpecificArgs specific_args; }; -std::optional getURLBasedDataSourceConfiguration(const ASTs & args, ContextPtr context); - std::optional getURLBasedDataSourceConfiguration( const Poco::Util::AbstractConfiguration & dict_config, const String & dict_config_prefix, ContextPtr context); diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 7d1eef1e47c..4ebe31eb1a9 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -247,24 +248,11 @@ void registerStorageExternalDistributed(StorageFactory & factory) if (engine_name == "URL") { - URLBasedDataSourceConfiguration configuration; - if (auto named_collection = getURLBasedDataSourceConfiguration(inner_engine_args, args.getLocalContext())) + StorageURL::Configuration configuration; + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - auto [common_configuration, storage_specific_args] = named_collection.value(); - configuration.set(common_configuration); - - for (const auto & [name, value] : storage_specific_args) - { - if (name == "description") - cluster_description = checkAndGetLiteralArgument(value, "cluster_description"); - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unknown key-value argument {} for table engine URL", name); - } - - if (cluster_description.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Engine ExternalDistribued must have `description` key-value argument or named collection parameter"); + StorageURL::processNamedCollectionResult(configuration, *named_collection); + StorageURL::collectHeaders(engine_args, configuration.headers, args.getLocalContext()); } else { diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 6b875c2d138..ef0d6fe1706 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -63,6 +63,7 @@ static const std::unordered_set optional_configuration_keys = "compression_method", "structure", "filename", + "description", "headers.header.name", "headers.header.value", }; From 2ce5af421e3181cfd0aa01a995742d09db4121e4 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 17 Dec 2022 00:49:07 +0100 Subject: [PATCH 07/88] Replace old named collections code for mongo --- .../ExternalDataSourceConfiguration.h | 6 --- src/Storages/StorageMongoDB.cpp | 44 ++++++++++++------- src/Storages/StorageMongoDB.h | 13 +++++- src/Storages/StorageS3.cpp | 2 +- src/TableFunctions/TableFunctionMongoDB.h | 2 +- 5 files changed, 42 insertions(+), 25 deletions(-) diff --git a/src/Storages/ExternalDataSourceConfiguration.h b/src/Storages/ExternalDataSourceConfiguration.h index ed6d7b3447f..65ef7d28b1b 100644 --- a/src/Storages/ExternalDataSourceConfiguration.h +++ b/src/Storages/ExternalDataSourceConfiguration.h @@ -46,12 +46,6 @@ struct StorageMySQLConfiguration : ExternalDataSourceConfiguration String on_duplicate_clause; }; -struct StorageMongoDBConfiguration : ExternalDataSourceConfiguration -{ - String options; -}; - - using StorageSpecificArgs = std::vector>; struct ExternalDataSourceInfo diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 3ae9c974770..fac7e35eb8f 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -11,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -27,6 +29,19 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +static const std::unordered_set required_configuration_keys = { + "host", + "port", + "username", + "passowrd", + "database", + "collection", + "table", +}; +static const std::unordered_set optional_configuration_keys = { + "options", +}; + StorageMongoDB::StorageMongoDB( const StorageID & table_id_, const std::string & host_, @@ -172,24 +187,21 @@ SinkToStoragePtr StorageMongoDB::write(const ASTPtr & /* query */, const Storage return std::make_shared(collection_name, database_name, metadata_snapshot, connection); } -StorageMongoDBConfiguration StorageMongoDB::getConfiguration(ASTs engine_args, ContextPtr context) +StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, ContextPtr context) { - StorageMongoDBConfiguration configuration; - if (auto named_collection = getExternalDataSourceConfiguration(engine_args, context)) - { - auto [common_configuration, storage_specific_args, _] = named_collection.value(); - configuration.set(common_configuration); + Configuration configuration; - for (const auto & [arg_name, arg_value] : storage_specific_args) - { - if (arg_name == "options") - configuration.options = checkAndGetLiteralArgument(arg_value, "options"); - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unexpected key-value argument." - "Got: {}, but expected one of:" - "host, port, username, password, database, table, options.", arg_name); - } + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) + { + validateNamedCollection(*named_collection, required_configuration_keys, optional_configuration_keys); + + configuration.host = named_collection->get("host"); + configuration.port = static_cast(named_collection->get("port")); + configuration.username = named_collection->get("username"); + configuration.password = named_collection->get("password"); + configuration.database = named_collection->get("database"); + configuration.table = named_collection->getOrDefault("collection", named_collection->get("table")); + configuration.options = named_collection->getOrDefault("options", ""); } else { diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index 04fb759133a..941e9fcf4b0 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -44,7 +44,18 @@ public: const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; - static StorageMongoDBConfiguration getConfiguration(ASTs engine_args, ContextPtr context); + struct Configuration + { + std::string host; + UInt16 port; + std::string username; + std::string password; + std::string database; + std::string table; + std::string options; + }; + + static Configuration getConfiguration(ASTs engine_args, ContextPtr context); private: void connectIfNotConnected(); diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 2646d577770..ea48b3c8c86 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -80,7 +80,7 @@ static const String PARTITION_ID_WILDCARD = "{_partition_id}"; static const std::unordered_set required_configuration_keys = { "url", }; -static std::unordered_set optional_configuration_keys = { +static const std::unordered_set optional_configuration_keys = { "format", "compression", "compression_method", diff --git a/src/TableFunctions/TableFunctionMongoDB.h b/src/TableFunctions/TableFunctionMongoDB.h index dd62bf4b2b4..b5033b2d654 100644 --- a/src/TableFunctions/TableFunctionMongoDB.h +++ b/src/TableFunctions/TableFunctionMongoDB.h @@ -24,7 +24,7 @@ private: ColumnsDescription getActualTableStructure(ContextPtr context) const override; void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; - std::optional configuration; + std::optional configuration; String structure; }; From 49386d19412e45298ee6985d37dbe865618ef707 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 17 Dec 2022 00:55:47 +0100 Subject: [PATCH 08/88] Replace old named collections code for meilisearch --- src/Storages/MeiliSearch/StorageMeiliSearch.cpp | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp index 30d49edbb10..3eca27ef3a8 100644 --- a/src/Storages/MeiliSearch/StorageMeiliSearch.cpp +++ b/src/Storages/MeiliSearch/StorageMeiliSearch.cpp @@ -14,8 +14,10 @@ #include #include #include +#include #include #include +#include namespace DB { @@ -127,13 +129,13 @@ SinkToStoragePtr StorageMeiliSearch::write(const ASTPtr & /*query*/, const Stora MeiliSearchConfiguration StorageMeiliSearch::getConfiguration(ASTs engine_args, ContextPtr context) { - if (auto named_collection = getExternalDataSourceConfiguration(engine_args, context)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - auto [common_configuration, storage_specific_args, _] = named_collection.value(); + validateNamedCollection(*named_collection, {"url", "index"}, {"key"}); - String url = common_configuration.addresses_expr; - String index = common_configuration.table; - String key = common_configuration.password; + String url = named_collection->get("url"); + String index = named_collection->get("index"); + String key = named_collection->getOrDefault("key", ""); if (url.empty() || index.empty()) { From 6bb3d06c8899dd37b1b624542b8bec236dff95d6 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 17 Dec 2022 01:30:55 +0100 Subject: [PATCH 09/88] Replace old named collections code for postgresql --- src/Core/PostgreSQL/PoolWithFailover.cpp | 2 +- src/Core/PostgreSQL/PoolWithFailover.h | 3 +- src/Databases/DatabaseFactory.cpp | 53 +++++++++++-------- .../PostgreSQL/DatabasePostgreSQL.cpp | 2 +- src/Databases/PostgreSQL/DatabasePostgreSQL.h | 4 +- src/Storages/StorageExternalDistributed.cpp | 9 +++- src/Storages/StoragePostgreSQL.cpp | 41 ++++++++------ src/Storages/StoragePostgreSQL.h | 24 ++++++++- .../TableFunctionPostgreSQL.cpp | 1 - src/TableFunctions/TableFunctionPostgreSQL.h | 3 +- 10 files changed, 91 insertions(+), 51 deletions(-) diff --git a/src/Core/PostgreSQL/PoolWithFailover.cpp b/src/Core/PostgreSQL/PoolWithFailover.cpp index 1bac17de579..79b1068b1f0 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.cpp +++ b/src/Core/PostgreSQL/PoolWithFailover.cpp @@ -45,7 +45,7 @@ PoolWithFailover::PoolWithFailover( } PoolWithFailover::PoolWithFailover( - const DB::StoragePostgreSQLConfiguration & configuration, + const DB::StoragePostgreSQL::Configuration & configuration, size_t pool_size, size_t pool_wait_timeout_, size_t max_tries_, diff --git a/src/Core/PostgreSQL/PoolWithFailover.h b/src/Core/PostgreSQL/PoolWithFailover.h index f8525684e23..afef2933d29 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.h +++ b/src/Core/PostgreSQL/PoolWithFailover.h @@ -10,6 +10,7 @@ #include #include #include +#include static constexpr inline auto POSTGRESQL_POOL_DEFAULT_SIZE = 16; @@ -33,7 +34,7 @@ public: bool auto_close_connection_); explicit PoolWithFailover( - const DB::StoragePostgreSQLConfiguration & configuration, + const DB::StoragePostgreSQL::Configuration & configuration, size_t pool_size, size_t pool_wait_timeout, size_t max_tries_, diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 5ce1dee4702..34e1f71c2e7 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -14,6 +14,8 @@ #include #include #include +#include +#include #include #include #include @@ -322,25 +324,24 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String ASTs & engine_args = engine->arguments->children; auto use_table_cache = false; - StoragePostgreSQLConfiguration configuration; + StoragePostgreSQL::Configuration configuration; - if (auto named_collection = getExternalDataSourceConfiguration(engine_args, context, true)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - auto [common_configuration, storage_specific_args, _] = named_collection.value(); + validateNamedCollection( + *named_collection, + {"host", "port", "username", "password", "database", "table"}, + {"schema", "on_conflict", "use_table_cache"}); - configuration.set(common_configuration); - configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; - - for (const auto & [arg_name, arg_value] : storage_specific_args) - { - if (arg_name == "use_table_cache") - use_table_cache = true; - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unexpected key-value argument." - "Got: {}, but expected one of:" - "host, port, username, password, database, schema, use_table_cache.", arg_name); - } + configuration.host = named_collection->get("host"); + configuration.port = static_cast(named_collection->get("port")); + configuration.username = named_collection->get("username"); + configuration.password = named_collection->get("password"); + configuration.database = named_collection->get("database"); + configuration.table = named_collection->get("table"); + configuration.schema = named_collection->getOrDefault("schema", ""); + configuration.on_conflict = named_collection->getOrDefault("on_conflict", ""); + use_table_cache = named_collection->getOrDefault("use_tables_cache", 0); } else { @@ -398,16 +399,22 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String throw Exception(ErrorCodes::BAD_ARGUMENTS, "Engine `{}` must have arguments", engine_name); ASTs & engine_args = engine->arguments->children; - StoragePostgreSQLConfiguration configuration; + StoragePostgreSQL::Configuration configuration; - if (auto named_collection = getExternalDataSourceConfiguration(engine_args, context, true)) + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - auto [common_configuration, storage_specific_args, _] = named_collection.value(); - configuration.set(common_configuration); + validateNamedCollection( + *named_collection, + {"host", "port", "username", "password", "database", "table"}, + {"schema"}); - if (!storage_specific_args.empty()) - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "MaterializedPostgreSQL Database requires only `host`, `port`, `database_name`, `username`, `password`."); + configuration.host = named_collection->get("host"); + configuration.port = static_cast(named_collection->get("port")); + configuration.username = named_collection->get("username"); + configuration.password = named_collection->get("password"); + configuration.database = named_collection->get("database"); + configuration.table = named_collection->get("table"); + configuration.schema = named_collection->getOrDefault("schema", ""); } else { diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp index 79133d2e2fd..e947b6b768d 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.cpp @@ -41,7 +41,7 @@ DatabasePostgreSQL::DatabasePostgreSQL( const String & metadata_path_, const ASTStorage * database_engine_define_, const String & dbname_, - const StoragePostgreSQLConfiguration & configuration_, + const StoragePostgreSQL::Configuration & configuration_, postgres::PoolWithFailoverPtr pool_, bool cache_tables_) : IDatabase(dbname_) diff --git a/src/Databases/PostgreSQL/DatabasePostgreSQL.h b/src/Databases/PostgreSQL/DatabasePostgreSQL.h index 18e4c949121..074a8728d0a 100644 --- a/src/Databases/PostgreSQL/DatabasePostgreSQL.h +++ b/src/Databases/PostgreSQL/DatabasePostgreSQL.h @@ -30,7 +30,7 @@ public: const String & metadata_path_, const ASTStorage * database_engine_define, const String & dbname_, - const StoragePostgreSQLConfiguration & configuration, + const StoragePostgreSQL::Configuration & configuration, postgres::PoolWithFailoverPtr pool_, bool cache_tables_); @@ -67,7 +67,7 @@ protected: private: String metadata_path; ASTPtr database_engine_define; - StoragePostgreSQLConfiguration configuration; + StoragePostgreSQL::Configuration configuration; postgres::PoolWithFailoverPtr pool; const bool cache_tables; diff --git a/src/Storages/StorageExternalDistributed.cpp b/src/Storages/StorageExternalDistributed.cpp index 4ebe31eb1a9..a40ae12b2d1 100644 --- a/src/Storages/StorageExternalDistributed.cpp +++ b/src/Storages/StorageExternalDistributed.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -89,9 +90,13 @@ StorageExternalDistributed::StorageExternalDistributed( case ExternalStorageEngine::PostgreSQL: { addresses = parseRemoteDescriptionForExternalDatabase(shard_description, max_addresses, 5432); - StoragePostgreSQLConfiguration postgres_conf; - postgres_conf.set(configuration); + StoragePostgreSQL::Configuration postgres_conf; postgres_conf.addresses = addresses; + postgres_conf.username = configuration.username; + postgres_conf.password = configuration.password; + postgres_conf.database = configuration.database; + postgres_conf.table = configuration.table; + postgres_conf.schema = configuration.schema; const auto & settings = context->getSettingsRef(); auto pool = std::make_shared( diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 6cf4e458438..4d80c0a81ae 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -6,8 +6,11 @@ #include #include #include -#include #include +#include + +#include +#include #include #include @@ -38,6 +41,7 @@ #include #include #include +#include namespace DB @@ -385,26 +389,29 @@ SinkToStoragePtr StoragePostgreSQL::write( } -StoragePostgreSQLConfiguration StoragePostgreSQL::getConfiguration(ASTs engine_args, ContextPtr context) +StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine_args, ContextPtr context) { - StoragePostgreSQLConfiguration configuration; - if (auto named_collection = getExternalDataSourceConfiguration(engine_args, context)) + StoragePostgreSQL::Configuration configuration; + if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - auto [common_configuration, storage_specific_args, _] = named_collection.value(); + validateNamedCollection( + *named_collection, + {"username", "password", "database", "table"}, + {"schema", "on_conflict", "addresses_expr", "host", "port"}); - configuration.set(common_configuration); - configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; - - for (const auto & [arg_name, arg_value] : storage_specific_args) + configuration.addresses_expr = named_collection->getOrDefault("addresses_expr", ""); + if (configuration.addresses_expr.empty()) { - if (arg_name == "on_conflict") - configuration.on_conflict = checkAndGetLiteralArgument(arg_value, "on_conflict"); - else - throw Exception(ErrorCodes::BAD_ARGUMENTS, - "Unexpected key-value argument." - "Got: {}, but expected one of:" - "host, port, username, password, database, table, schema, on_conflict.", arg_name); + configuration.host = named_collection->get("host"); + configuration.port = static_cast(named_collection->get("port")); } + + configuration.username = named_collection->get("username"); + configuration.password = named_collection->get("password"); + configuration.database = named_collection->get("database"); + configuration.table = named_collection->get("table"); + configuration.schema = named_collection->getOrDefault("schema", ""); + configuration.on_conflict = named_collection->getOrDefault("on_conflict", ""); } else { @@ -425,7 +432,7 @@ StoragePostgreSQLConfiguration StoragePostgreSQL::getConfiguration(ASTs engine_a configuration.host = configuration.addresses[0].first; configuration.port = configuration.addresses[0].second; } - configuration.database = checkAndGetLiteralArgument(engine_args[1], "host:port"); + configuration.database = checkAndGetLiteralArgument(engine_args[1], "database"); configuration.table = checkAndGetLiteralArgument(engine_args[2], "table"); configuration.username = checkAndGetLiteralArgument(engine_args[3], "username"); configuration.password = checkAndGetLiteralArgument(engine_args[4], "password"); diff --git a/src/Storages/StoragePostgreSQL.h b/src/Storages/StoragePostgreSQL.h index 97c62daa50f..f3df24ebd35 100644 --- a/src/Storages/StoragePostgreSQL.h +++ b/src/Storages/StoragePostgreSQL.h @@ -5,7 +5,6 @@ #if USE_LIBPQXX #include #include -#include #include namespace Poco @@ -13,6 +12,12 @@ namespace Poco class Logger; } +namespace postgres +{ +class PoolWithFailover; +using PoolWithFailoverPtr = std::shared_ptr; +} + namespace DB { @@ -42,7 +47,22 @@ public: SinkToStoragePtr write(const ASTPtr & query, const StorageMetadataPtr & /*metadata_snapshot*/, ContextPtr context) override; - static StoragePostgreSQLConfiguration getConfiguration(ASTs engine_args, ContextPtr context); + struct Configuration + { + String host; + UInt16 port = 0; + String username = "default"; + String password; + String database; + String table; + String schema; + String on_conflict; + + std::vector> addresses; /// Failover replicas. + String addresses_expr; + }; + + static Configuration getConfiguration(ASTs engine_args, ContextPtr context); private: String remote_table_name; diff --git a/src/TableFunctions/TableFunctionPostgreSQL.cpp b/src/TableFunctions/TableFunctionPostgreSQL.cpp index d61140e1a07..2edfe82c708 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.cpp +++ b/src/TableFunctions/TableFunctionPostgreSQL.cpp @@ -2,7 +2,6 @@ #if USE_LIBPQXX #include -#include #include #include diff --git a/src/TableFunctions/TableFunctionPostgreSQL.h b/src/TableFunctions/TableFunctionPostgreSQL.h index a5971b18d2f..9f10e1c180e 100644 --- a/src/TableFunctions/TableFunctionPostgreSQL.h +++ b/src/TableFunctions/TableFunctionPostgreSQL.h @@ -5,6 +5,7 @@ #include #include #include +#include namespace DB @@ -27,7 +28,7 @@ private: void parseArguments(const ASTPtr & ast_function, ContextPtr context) override; postgres::PoolWithFailoverPtr connection_pool; - std::optional configuration; + std::optional configuration; }; } From f689fa5964f7041413586848c7520a1ee986e603 Mon Sep 17 00:00:00 2001 From: kssenii Date: Sat, 17 Dec 2022 01:34:19 +0100 Subject: [PATCH 10/88] Fix style check --- src/IO/S3Common.h | 6 +++--- src/Storages/StorageMongoDB.cpp | 19 ++++--------------- src/Storages/StoragePostgreSQL.cpp | 1 - 3 files changed, 7 insertions(+), 19 deletions(-) diff --git a/src/IO/S3Common.h b/src/IO/S3Common.h index 44a1b1a30d8..abd0f083b0b 100644 --- a/src/IO/S3Common.h +++ b/src/IO/S3Common.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include @@ -17,7 +18,6 @@ #include #include -#include namespace Aws::S3 { @@ -80,7 +80,7 @@ public: const String & access_key_id, const String & secret_access_key, const String & server_side_encryption_customer_key_base64, - DB::HTTPHeaderEntries headers, + HTTPHeaderEntries headers, bool use_environment_credentials, bool use_insecure_imds_request); @@ -154,7 +154,7 @@ struct AuthSettings std::string region; std::string server_side_encryption_customer_key_base64; - DB::HTTPHeaderEntries headers; + HTTPHeaderEntries headers; std::optional use_environment_credentials; std::optional use_insecure_imds_request; diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index fac7e35eb8f..31e59cf83dc 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -26,22 +26,8 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int MONGODB_CANNOT_AUTHENTICATE; - extern const int BAD_ARGUMENTS; } -static const std::unordered_set required_configuration_keys = { - "host", - "port", - "username", - "passowrd", - "database", - "collection", - "table", -}; -static const std::unordered_set optional_configuration_keys = { - "options", -}; - StorageMongoDB::StorageMongoDB( const StorageID & table_id_, const std::string & host_, @@ -193,7 +179,10 @@ StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, if (auto named_collection = tryGetNamedCollectionWithOverrides(engine_args)) { - validateNamedCollection(*named_collection, required_configuration_keys, optional_configuration_keys); + validateNamedCollection( + *named_collection, + {"host", "port", "username", "password", "database", "collection", "table"}, + {"options"}); configuration.host = named_collection->get("host"); configuration.port = static_cast(named_collection->get("port")); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 4d80c0a81ae..dd9d4340722 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -50,7 +50,6 @@ namespace DB namespace ErrorCodes { extern const int NOT_IMPLEMENTED; - extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; } From 326747b555d6248a31fbc417e9242c7f6b65c6a2 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 19 Dec 2022 12:16:50 +0100 Subject: [PATCH 11/88] Fix tests --- src/Storages/StorageURL.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index ef0d6fe1706..6540e797e5c 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -1010,11 +1010,8 @@ ASTs::iterator StorageURL::collectHeaders( for (auto arg_it = url_function_args.begin(); arg_it != url_function_args.end(); ++arg_it) { const auto * headers_ast_function = (*arg_it)->as(); - if (headers_ast_function) + if (headers_ast_function && headers_ast_function->name == "headers") { - if (headers_ast_function->name != "headers") - continue; - if (headers_it != url_function_args.end()) throw Exception( ErrorCodes::BAD_ARGUMENTS, @@ -1057,6 +1054,9 @@ ASTs::iterator StorageURL::collectHeaders( continue; } + if (headers_ast_function && headers_ast_function->name == "equals") + continue; + (*arg_it) = evaluateConstantExpressionOrIdentifierAsLiteral((*arg_it), context); } From 8d9bf775885ac07f4b7f66f9baf948c0f27a3f06 Mon Sep 17 00:00:00 2001 From: kssenii Date: Mon, 19 Dec 2022 13:56:23 +0100 Subject: [PATCH 12/88] Fix tests --- src/Storages/StoragePostgreSQL.cpp | 4 ++-- src/Storages/StorageURL.cpp | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index dd9d4340722..202e0f55c3b 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -395,7 +395,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine { validateNamedCollection( *named_collection, - {"username", "password", "database", "table"}, + {"username", "user", "password", "database", "table"}, {"schema", "on_conflict", "addresses_expr", "host", "port"}); configuration.addresses_expr = named_collection->getOrDefault("addresses_expr", ""); @@ -405,7 +405,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine configuration.port = static_cast(named_collection->get("port")); } - configuration.username = named_collection->get("username"); + configuration.username = named_collection->getOrDefault("username", named_collection->get("user")); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); configuration.table = named_collection->get("table"); diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index 6540e797e5c..9778d2a697c 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -63,6 +63,8 @@ static const std::unordered_set optional_configuration_keys = "compression_method", "structure", "filename", + "method", + "http_method", "description", "headers.header.name", "headers.header.value", @@ -1070,7 +1072,7 @@ void StorageURL::processNamedCollectionResult(Configuration & configuration, con configuration.url = collection.get("url"); configuration.headers = getHeadersFromNamedCollection(collection); - configuration.http_method = collection.getOrDefault("http_method", ""); + configuration.http_method = collection.getOrDefault("http_method", collection.getOrDefault("method", "")); if (!configuration.http_method.empty() && configuration.http_method != Poco::Net::HTTPRequest::HTTP_POST && configuration.http_method != Poco::Net::HTTPRequest::HTTP_PUT) throw Exception( From 56fa477db6903193b9873941e2ee82ecae5361c3 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 20 Dec 2022 18:51:55 +0100 Subject: [PATCH 13/88] Disallow array join in mutations Implementation: * Added a new parameter to ActionsVisitor::Data disallow_arrayjoin, which is set by MutationsIterator when it appends expression. * ActionsVisitor uses disallow_arrayjoin and throws error when its used with mutations. Testing: * Added test for the same 02504_disallow_arrayjoin_in_mutations.sql. --- src/Interpreters/ActionsVisitor.cpp | 7 ++++++- src/Interpreters/ActionsVisitor.h | 4 +++- src/Interpreters/ExpressionAnalyzer.cpp | 10 ++++++---- src/Interpreters/ExpressionAnalyzer.h | 4 ++-- src/Interpreters/MutationsInterpreter.cpp | 5 +++-- .../02504_disallow_arrayjoin_in_mutations.reference | 3 +++ .../02504_disallow_arrayjoin_in_mutations.sql | 9 +++++++++ 7 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.reference create mode 100644 tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.sql diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 9a0d33b19fc..5dfb252fa18 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -535,7 +535,8 @@ ActionsMatcher::Data::Data( bool only_consts_, bool create_source_for_in_, AggregationKeysInfo aggregation_keys_info_, - bool build_expression_with_window_functions_) + bool build_expression_with_window_functions_, + bool disallow_arrayjoin_) : WithContext(context_) , set_size_limit(set_size_limit_) , subquery_depth(subquery_depth_) @@ -549,6 +550,7 @@ ActionsMatcher::Data::Data( , actions_stack(std::move(actions_dag), context_) , aggregation_keys_info(aggregation_keys_info_) , build_expression_with_window_functions(build_expression_with_window_functions_) + , disallow_arrayjoin(disallow_arrayjoin_) , next_unique_suffix(actions_stack.getLastActions().getOutputs().size() + 1) { } @@ -887,6 +889,9 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & /// Function arrayJoin. if (node.name == "arrayJoin") { + if (data.disallow_arrayjoin) + throw Exception("arrayJoin is disallowed in mutations", ErrorCodes::UNEXPECTED_EXPRESSION); + if (node.arguments->children.size() != 1) throw Exception("arrayJoin requires exactly 1 argument", ErrorCodes::TYPE_MISMATCH); diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index fea013fd075..16b32162bfe 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -134,6 +134,7 @@ public: ScopeStack actions_stack; AggregationKeysInfo aggregation_keys_info; bool build_expression_with_window_functions; + bool disallow_arrayjoin; /* * Remember the last unique column suffix to avoid quadratic behavior @@ -154,7 +155,8 @@ public: bool only_consts_, bool create_source_for_in_, AggregationKeysInfo aggregation_keys_info_, - bool build_expression_with_window_functions_ = false); + bool build_expression_with_window_functions_ = false, + bool disallow_arrayjoin = false); /// Does result of the calculation already exists in the block. bool hasColumn(const String & column_name) const; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 22229c0d6c2..46ecc3f0a42 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -538,7 +538,7 @@ void SelectQueryExpressionAnalyzer::makeSetsForIndex(const ASTPtr & node) } -void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts) +void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts, bool disallow_arrayJoin) { LogAST log; ActionsVisitor::Data visitor_data( @@ -552,7 +552,9 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_ false /* no_makeset */, only_consts, !isRemoteStorage() /* create_source_for_in */, - getAggregationKeysInfo()); + getAggregationKeysInfo(), + false /*build_expression_with_window_functions */, + disallow_arrayJoin); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -1711,10 +1713,10 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio } -void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types) +void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types, bool disallow_arrayJoin) { ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns()); - getRootActions(expr, only_types, step.actions()); + getRootActions(expr, only_types, step.actions(), false /* only_consts */, disallow_arrayJoin); step.addRequiredOutput(expr->getColumnName()); } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index ddb41a00f84..cb376ab6326 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -110,7 +110,7 @@ public: ~ExpressionAnalyzer(); - void appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types); + void appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types, bool disallow_arrayJoin = false); /// If `ast` is not a SELECT query, just gets all the actions to evaluate the expression. /// If add_aliases, only the calculated values in the desired order and add aliases. @@ -175,7 +175,7 @@ protected: ArrayJoinActionPtr addMultipleArrayJoinAction(ActionsDAGPtr & actions, bool is_left) const; - void getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts = false); + void getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts = false, bool disallow_arrayJoin = false); /** Similar to getRootActions but do not make sets when analyzing IN functions. It's used in * analyzeAggregation which happens earlier than analyzing PREWHERE and WHERE. If we did, the diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 26b8bce1f4a..89e85b6a7da 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -861,7 +861,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & { if (!actions_chain.steps.empty()) actions_chain.addStep(); - stage.analyzer->appendExpression(actions_chain, ast, dry_run); + stage.analyzer->appendExpression(actions_chain, ast, dry_run, true /* disallow_arrayJoin */); stage.filter_column_names.push_back(ast->getColumnName()); } @@ -871,7 +871,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & actions_chain.addStep(); for (const auto & kv : stage.column_to_updated) - stage.analyzer->appendExpression(actions_chain, kv.second, dry_run); + stage.analyzer->appendExpression(actions_chain, kv.second, dry_run, true /* disallow_arrayJoin */); auto & actions = actions_chain.getLastStep().actions(); @@ -884,6 +884,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & } } + /// Remove all intermediate columns. actions_chain.addStep(); actions_chain.getLastStep().required_output.clear(); diff --git a/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.reference b/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.reference new file mode 100644 index 00000000000..0cfb83aa2f2 --- /dev/null +++ b/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.reference @@ -0,0 +1,3 @@ +1 1 +2 2 +3 3 diff --git a/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.sql b/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.sql new file mode 100644 index 00000000000..3cf2c83ffe5 --- /dev/null +++ b/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.sql @@ -0,0 +1,9 @@ +DROP TABLE IF EXISTS test_02504; + +CREATE TABLE test_02504 (`a` UInt32,`b` UInt32) ENGINE = MergeTree ORDER BY a; +INSERT INTO test_02504 values (1, 1) (2, 2), (3, 3); +SELECT * FROM test_02504; + +ALTER TABLE test_02504 UPDATE b = 33 WHERE arrayJoin([1, 2]) = a; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT} + +DROP TABLE test_02504; \ No newline at end of file From f9e06cc93f119a0f197d1cbff58ee6c117540afb Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 20 Dec 2022 19:33:51 +0100 Subject: [PATCH 14/88] Removed extra whitespace - Disallow array join in mutations --- src/Interpreters/ExpressionAnalyzer.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 46ecc3f0a42..7478c749d7d 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1716,7 +1716,7 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types, bool disallow_arrayJoin) { ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns()); - getRootActions(expr, only_types, step.actions(), false /* only_consts */, disallow_arrayJoin); + getRootActions(expr, only_types, step.actions(), false /* only_consts */, disallow_arrayJoin); step.addRequiredOutput(expr->getColumnName()); } From d6aded79c7a3f1c4886c113f5a971b135b2efe7c Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Tue, 20 Dec 2022 20:08:58 +0100 Subject: [PATCH 15/88] Fixed expected error in test - Disallow array join in mutations --- .../0_stateless/02504_disallow_arrayjoin_in_mutations.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.sql b/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.sql index 3cf2c83ffe5..d261a71d912 100644 --- a/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.sql +++ b/tests/queries/0_stateless/02504_disallow_arrayjoin_in_mutations.sql @@ -4,6 +4,6 @@ CREATE TABLE test_02504 (`a` UInt32,`b` UInt32) ENGINE = MergeTree ORDER BY a; INSERT INTO test_02504 values (1, 1) (2, 2), (3, 3); SELECT * FROM test_02504; -ALTER TABLE test_02504 UPDATE b = 33 WHERE arrayJoin([1, 2]) = a; -- { serverError ILLEGAL_TYPE_OF_ARGUMENT} +ALTER TABLE test_02504 UPDATE b = 33 WHERE arrayJoin([1, 2]) = a; -- { serverError UNEXPECTED_EXPRESSION} DROP TABLE test_02504; \ No newline at end of file From 1d75f740d7730ecf250f87227b2556c12df8b3be Mon Sep 17 00:00:00 2001 From: kssenii Date: Tue, 20 Dec 2022 22:33:54 +0100 Subject: [PATCH 16/88] Fix tests --- .../NamedCollectionConfiguration.cpp | 7 ++++++ .../NamedCollectionConfiguration.h | 4 ++++ .../NamedCollections/NamedCollections.cpp | 22 +++++++++++++------ src/Databases/DatabaseFactory.cpp | 8 +++---- src/Storages/StorageMongoDB.cpp | 4 ++-- src/Storages/StoragePostgreSQL.cpp | 6 ++--- src/Storages/StorageS3.cpp | 3 +++ src/Storages/StorageS3Settings.cpp | 2 +- .../configs/named_collection.xml | 8 +++---- 9 files changed, 42 insertions(+), 22 deletions(-) diff --git a/src/Common/NamedCollections/NamedCollectionConfiguration.cpp b/src/Common/NamedCollections/NamedCollectionConfiguration.cpp index 7fd1635c615..7cc007d739b 100644 --- a/src/Common/NamedCollections/NamedCollectionConfiguration.cpp +++ b/src/Common/NamedCollections/NamedCollectionConfiguration.cpp @@ -16,6 +16,13 @@ namespace ErrorCodes namespace NamedCollectionConfiguration { +bool hasConfigValue( + const Poco::Util::AbstractConfiguration & config, + const std::string & path) +{ + return config.has(path); +} + template T getConfigValue( const Poco::Util::AbstractConfiguration & config, const std::string & path) diff --git a/src/Common/NamedCollections/NamedCollectionConfiguration.h b/src/Common/NamedCollections/NamedCollectionConfiguration.h index d545ecfb3e6..bde3bf09d77 100644 --- a/src/Common/NamedCollections/NamedCollectionConfiguration.h +++ b/src/Common/NamedCollections/NamedCollectionConfiguration.h @@ -14,6 +14,10 @@ namespace NamedCollectionConfiguration ConfigurationPtr createEmptyConfiguration(const std::string & root_name); +bool hasConfigValue( + const Poco::Util::AbstractConfiguration & config, + const std::string & path); + template T getConfigValue( const Poco::Util::AbstractConfiguration & config, const std::string & path); diff --git a/src/Common/NamedCollections/NamedCollections.cpp b/src/Common/NamedCollections/NamedCollections.cpp index 5876a8567f4..760c1ff91a8 100644 --- a/src/Common/NamedCollections/NamedCollections.cpp +++ b/src/Common/NamedCollections/NamedCollections.cpp @@ -231,15 +231,23 @@ public: Keys getKeys(ssize_t depth, const std::string & prefix) const { - if (depth == -1) - { - /// Return all keys with full depth. - return keys; - } - std::queue enumerate_input; - if (!prefix.empty()) + + if (prefix.empty()) + { + if (depth == -1) + { + /// Return all keys with full depth. + return keys; + } + } + else + { + if (!Configuration::hasConfigValue(*config, prefix)) + return {}; + enumerate_input.push(prefix); + } Keys result; Configuration::listKeys(*config, enumerate_input, result, depth); diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 34e1f71c2e7..6d834fcc25e 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -330,12 +330,12 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String { validateNamedCollection( *named_collection, - {"host", "port", "username", "password", "database", "table"}, + {"host", "port", "user", "password", "database", "table"}, {"schema", "on_conflict", "use_table_cache"}); configuration.host = named_collection->get("host"); configuration.port = static_cast(named_collection->get("port")); - configuration.username = named_collection->get("username"); + configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); configuration.table = named_collection->get("table"); @@ -405,12 +405,12 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String { validateNamedCollection( *named_collection, - {"host", "port", "username", "password", "database", "table"}, + {"host", "port", "user", "password", "database", "table"}, {"schema"}); configuration.host = named_collection->get("host"); configuration.port = static_cast(named_collection->get("port")); - configuration.username = named_collection->get("username"); + configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); configuration.table = named_collection->get("table"); diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 31e59cf83dc..871154b4b5b 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -181,12 +181,12 @@ StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, { validateNamedCollection( *named_collection, - {"host", "port", "username", "password", "database", "collection", "table"}, + {"host", "port", "user", "password", "database", "collection", "table"}, {"options"}); configuration.host = named_collection->get("host"); configuration.port = static_cast(named_collection->get("port")); - configuration.username = named_collection->get("username"); + configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); configuration.table = named_collection->getOrDefault("collection", named_collection->get("table")); diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index 202e0f55c3b..bfffac30084 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -395,7 +395,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine { validateNamedCollection( *named_collection, - {"username", "user", "password", "database", "table"}, + {"user", "password", "database", "table"}, {"schema", "on_conflict", "addresses_expr", "host", "port"}); configuration.addresses_expr = named_collection->getOrDefault("addresses_expr", ""); @@ -405,7 +405,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine configuration.port = static_cast(named_collection->get("port")); } - configuration.username = named_collection->getOrDefault("username", named_collection->get("user")); + configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); configuration.table = named_collection->get("table"); @@ -433,7 +433,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine } configuration.database = checkAndGetLiteralArgument(engine_args[1], "database"); configuration.table = checkAndGetLiteralArgument(engine_args[2], "table"); - configuration.username = checkAndGetLiteralArgument(engine_args[3], "username"); + configuration.username = checkAndGetLiteralArgument(engine_args[3], "user"); configuration.password = checkAndGetLiteralArgument(engine_args[4], "password"); if (engine_args.size() >= 6) diff --git a/src/Storages/StorageS3.cpp b/src/Storages/StorageS3.cpp index 1a7e53fcf0f..78cc3c32237 100644 --- a/src/Storages/StorageS3.cpp +++ b/src/Storages/StorageS3.cpp @@ -1242,6 +1242,9 @@ void StorageS3::processNamedCollectionResult(StorageS3Configuration & configurat { validateNamedCollection(collection, required_configuration_keys, optional_configuration_keys); + configuration.url = collection.get("url"); + + auto filename = collection.getOrDefault("filename", ""); if (!filename.empty()) configuration.url = std::filesystem::path(configuration.url) / filename; diff --git a/src/Storages/StorageS3Settings.cpp b/src/Storages/StorageS3Settings.cpp index 8c1974527b6..b38edd752ec 100644 --- a/src/Storages/StorageS3Settings.cpp +++ b/src/Storages/StorageS3Settings.cpp @@ -3,7 +3,7 @@ #include #include -#include +#include #include #include #include diff --git a/tests/integration/test_storage_meilisearch/configs/named_collection.xml b/tests/integration/test_storage_meilisearch/configs/named_collection.xml index cf55a837c1f..e11ee7d2161 100644 --- a/tests/integration/test_storage_meilisearch/configs/named_collection.xml +++ b/tests/integration/test_storage_meilisearch/configs/named_collection.xml @@ -2,22 +2,20 @@ MeiliSearch - http://meili1:7700 + http://meili1:7700 new_table
 - MeiliSearch - http://meili_secure:7700 + http://meili_secure:7700 new_table
password - MeiliSearch - http://meili_secure:7700 + http://meili_secure:7700 new_table
 From db90214c0bce521244e6aa1880fd90ebe988ce8e Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 21 Dec 2022 10:52:49 +0000 Subject: [PATCH 17/88] Make ASTSelectQuery::formatImpl() more robust Fixes #43049 --- src/Parsers/ASTSelectQuery.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index e0e3b1a90c1..46355e16575 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -113,7 +113,7 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (group_by_with_cube) s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH CUBE" << (s.hilite ? hilite_none : ""); - if (group_by_with_grouping_sets) + if (groupBy() && group_by_with_grouping_sets) { auto nested_frame = frame; nested_frame.surround_each_list_element_with_parens = true; From 05b70b0cb6598cce18c1b9342ce25c65f43217b6 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Wed, 21 Dec 2022 10:54:18 +0000 Subject: [PATCH 18/88] Update link to blog on fuzzing --- docs/en/development/tests.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/en/development/tests.md b/docs/en/development/tests.md index e6d5cf66de9..3b446e66e5b 100644 --- a/docs/en/development/tests.md +++ b/docs/en/development/tests.md @@ -202,7 +202,7 @@ Google OSS-Fuzz can be found at `docker/fuzz`. We also use simple fuzz test to generate random SQL queries and to check that the server does not die executing them. You can find it in `00746_sql_fuzzy.pl`. This test should be run continuously (overnight and longer). -We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. You can learn more about this fuzzer in [this blog article](https://clickhouse.com/blog/en/2021/fuzzing-clickhouse/). +We also use sophisticated AST-based query fuzzer that is able to find huge amount of corner cases. It does random permutations and substitutions in queries AST. It remembers AST nodes from previous tests to use them for fuzzing of subsequent tests while processing them in random order. You can learn more about this fuzzer in [this blog article](https://clickhouse.com/blog/fuzzing-click-house). ## Stress test From d3db1dd6a7ea5862e31c596943d1b550a6d873fc Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 22 Dec 2022 00:27:22 +0100 Subject: [PATCH 19/88] Fix tests --- src/Core/PostgreSQL/PoolWithFailover.cpp | 4 ++ src/Databases/DatabaseFactory.cpp | 6 +- src/Storages/NamedCollectionsHelpers.cpp | 49 --------------- src/Storages/NamedCollectionsHelpers.h | 59 ++++++++++++++++--- src/Storages/StorageMongoDB.cpp | 11 +++- src/Storages/StoragePostgreSQL.cpp | 12 +++- .../configs/named_collection.xml | 11 ++-- .../test_storage_meilisearch/test.py | 2 +- 8 files changed, 81 insertions(+), 73 deletions(-) diff --git a/src/Core/PostgreSQL/PoolWithFailover.cpp b/src/Core/PostgreSQL/PoolWithFailover.cpp index 79b1068b1f0..b6af20d1165 100644 --- a/src/Core/PostgreSQL/PoolWithFailover.cpp +++ b/src/Core/PostgreSQL/PoolWithFailover.cpp @@ -14,6 +14,7 @@ namespace DB namespace ErrorCodes { extern const int POSTGRESQL_CONNECTION_FAILURE; + extern const int LOGICAL_ERROR; } } @@ -70,6 +71,9 @@ ConnectionHolderPtr PoolWithFailover::get() { std::lock_guard lock(mutex); + if (replicas_with_priority.empty()) + throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "No address specified"); + DB::WriteBufferFromOwnString error_message; for (size_t try_idx = 0; try_idx < max_tries; ++try_idx) { diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index 6d834fcc25e..d0c1829f694 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -330,7 +330,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String { validateNamedCollection( *named_collection, - {"host", "port", "user", "password", "database", "table"}, + {"host", "port", "user", "password", "database"}, {"schema", "on_conflict", "use_table_cache"}); configuration.host = named_collection->get("host"); @@ -338,7 +338,6 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); - configuration.table = named_collection->get("table"); configuration.schema = named_collection->getOrDefault("schema", ""); configuration.on_conflict = named_collection->getOrDefault("on_conflict", ""); use_table_cache = named_collection->getOrDefault("use_tables_cache", 0); @@ -405,7 +404,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String { validateNamedCollection( *named_collection, - {"host", "port", "user", "password", "database", "table"}, + {"host", "port", "user", "password", "database"}, {"schema"}); configuration.host = named_collection->get("host"); @@ -413,7 +412,6 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); - configuration.table = named_collection->get("table"); configuration.schema = named_collection->getOrDefault("schema", ""); } else diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index b2935bb0fce..542865f2a80 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -8,11 +8,6 @@ namespace DB { -namespace ErrorCodes -{ - extern const int BAD_ARGUMENTS; -} - namespace { NamedCollectionPtr tryGetNamedCollectionFromASTs(ASTs asts) @@ -80,50 +75,6 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) return collection_copy; } -void validateNamedCollection( - const NamedCollection & collection, - const std::unordered_set & required_keys, - const std::unordered_set & optional_keys, - const std::vector & optional_regex_keys) -{ - const auto & keys = collection.getKeys(); - auto required_keys_copy = required_keys; - - for (const auto & key : keys) - { - auto it = required_keys_copy.find(key); - if (it != required_keys_copy.end()) - { - required_keys_copy.erase(it); - continue; - } - - if (optional_keys.contains(key)) - continue; - - auto match = std::find_if( - optional_regex_keys.begin(), optional_regex_keys.end(), - [&](const std::regex & regex) { return std::regex_search(key, regex); }) - != optional_regex_keys.end(); - - if (!match) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Unexpected key `{}` in named collection. Required keys: {}, optional keys: {}", - key, fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); - } - } - - if (!required_keys_copy.empty()) - { - throw Exception( - ErrorCodes::BAD_ARGUMENTS, - "Required keys ({}) are not specified. All required keys: {}, optional keys: {}", - fmt::join(required_keys_copy, ", "), fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); - } -} - HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collection) { HTTPHeaderEntries headers; diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 029f9f41ebf..b91df97782d 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -1,23 +1,68 @@ #pragma once #include #include -#include +#include #include #include +#include #include +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace DB { NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts); -void validateNamedCollection( - const NamedCollection & collection, - const std::unordered_set & required_keys, - const std::unordered_set & optional_keys, - const std::vector & optional_regex_keys = {}); - HTTPHeaderEntries getHeadersFromNamedCollection(const NamedCollection & collection); +template , + typename OptionalKeys = std::unordered_set> +void validateNamedCollection( + const NamedCollection & collection, + const RequiredKeys & required_keys, + const OptionalKeys & optional_keys, + const std::vector & optional_regex_keys = {}) +{ + const auto & keys = collection.getKeys(); + auto required_keys_copy = required_keys; + + for (const auto & key : keys) + { + if (required_keys_copy.contains(key)) + { + required_keys_copy.erase(key); + continue; + } + + if (optional_keys.contains(key)) + continue; + + auto match = std::find_if( + optional_regex_keys.begin(), optional_regex_keys.end(), + [&](const std::regex & regex) { return std::regex_search(key, regex); }) + != optional_regex_keys.end(); + + if (!match) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected key `{}` in named collection. Required keys: {}, optional keys: {}", + key, fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); + } + } + + if (!required_keys_copy.empty()) + { + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Required keys ({}) are not specified. All required keys: {}, optional keys: {}", + fmt::join(required_keys_copy, ", "), fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); + } +} + } diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index 871154b4b5b..1a96cd80323 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -173,6 +173,13 @@ SinkToStoragePtr StorageMongoDB::write(const ASTPtr & /* query */, const Storage return std::make_shared(collection_name, database_name, metadata_snapshot, connection); } +struct KeysCmp +{ + constexpr bool operator()(const auto & lhs, const auto & rhs) const + { + return lhs == rhs || ((lhs == "table") && (rhs == "collection")) || ((rhs == "table") && (lhs == "collection")); + } +}; StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, ContextPtr context) { Configuration configuration; @@ -181,7 +188,7 @@ StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, { validateNamedCollection( *named_collection, - {"host", "port", "user", "password", "database", "collection", "table"}, + std::unordered_multiset, KeysCmp>{"host", "port", "user", "password", "database", "collection", "table"}, {"options"}); configuration.host = named_collection->get("host"); @@ -189,7 +196,7 @@ StorageMongoDB::Configuration StorageMongoDB::getConfiguration(ASTs engine_args, configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); - configuration.table = named_collection->getOrDefault("collection", named_collection->get("table")); + configuration.table = named_collection->getOrDefault("collection", named_collection->getOrDefault("table", "")); configuration.options = named_collection->getOrDefault("options", ""); } else diff --git a/src/Storages/StoragePostgreSQL.cpp b/src/Storages/StoragePostgreSQL.cpp index bfffac30084..729ffdf0dde 100644 --- a/src/Storages/StoragePostgreSQL.cpp +++ b/src/Storages/StoragePostgreSQL.cpp @@ -403,6 +403,7 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine { configuration.host = named_collection->get("host"); configuration.port = static_cast(named_collection->get("port")); + configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; } configuration.username = named_collection->get("user"); @@ -415,9 +416,14 @@ StoragePostgreSQL::Configuration StoragePostgreSQL::getConfiguration(ASTs engine else { if (engine_args.size() < 5 || engine_args.size() > 7) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Storage PostgreSQL requires from 5 to 7 parameters: " - "PostgreSQL('host:port', 'database', 'table', 'username', 'password' [, 'schema', 'ON CONFLICT ...']. Got: {}", - engine_args.size()); + { + throw Exception( + ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, + "Storage PostgreSQL requires from 5 to 7 parameters: " + "PostgreSQL('host:port', 'database', 'table', 'username', 'password' " + "[, 'schema', 'ON CONFLICT ...']. Got: {}", + engine_args.size()); + } for (auto & engine_arg : engine_args) engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context); diff --git a/tests/integration/test_storage_meilisearch/configs/named_collection.xml b/tests/integration/test_storage_meilisearch/configs/named_collection.xml index e11ee7d2161..c20f556b3da 100644 --- a/tests/integration/test_storage_meilisearch/configs/named_collection.xml +++ b/tests/integration/test_storage_meilisearch/configs/named_collection.xml @@ -1,22 +1,19 @@ - MeiliSearch http://meili1:7700 - new_table
+ new_table - MeiliSearch http://meili_secure:7700 - new_table
- password + new_table + password - MeiliSearch http://meili_secure:7700 - new_table
+ new_table diff --git a/tests/integration/test_storage_meilisearch/test.py b/tests/integration/test_storage_meilisearch/test.py index fd11644cf31..32da7f20155 100644 --- a/tests/integration/test_storage_meilisearch/test.py +++ b/tests/integration/test_storage_meilisearch/test.py @@ -598,7 +598,7 @@ def test_named_collection_secure(started_cluster): ) node.query( - 'CREATE TABLE combine_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure_no_password, password="password" )' + 'CREATE TABLE combine_meili_table(id UInt64, data String) ENGINE = MeiliSearch( named_collection_for_meili_secure_no_password, key="password" )' ) assert node.query("SELECT COUNT() FROM simple_meili_table") == "100\n" From a58b8b8e6c4e5391e7227dcef31a9823de008e20 Mon Sep 17 00:00:00 2001 From: kssenii Date: Thu, 22 Dec 2022 12:15:34 +0100 Subject: [PATCH 20/88] Fix tests --- src/Databases/DatabaseFactory.cpp | 2 ++ src/Storages/NamedCollectionsHelpers.cpp | 13 +++++++++---- src/Storages/NamedCollectionsHelpers.h | 1 - .../test_postgresql_database_engine/test.py | 14 ++++++++++++++ tests/integration/test_storage_postgresql/test.py | 13 ------------- 5 files changed, 25 insertions(+), 18 deletions(-) diff --git a/src/Databases/DatabaseFactory.cpp b/src/Databases/DatabaseFactory.cpp index d0c1829f694..1d44fa4588f 100644 --- a/src/Databases/DatabaseFactory.cpp +++ b/src/Databases/DatabaseFactory.cpp @@ -335,6 +335,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String configuration.host = named_collection->get("host"); configuration.port = static_cast(named_collection->get("port")); + configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); @@ -409,6 +410,7 @@ DatabasePtr DatabaseFactory::getImpl(const ASTCreateQuery & create, const String configuration.host = named_collection->get("host"); configuration.port = static_cast(named_collection->get("port")); + configuration.addresses = {std::make_pair(configuration.host, configuration.port)}; configuration.username = named_collection->get("user"); configuration.password = named_collection->get("password"); configuration.database = named_collection->get("database"); diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index 542865f2a80..bd992a75d94 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -8,6 +8,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + namespace { NamedCollectionPtr tryGetNamedCollectionFromASTs(ASTs asts) @@ -62,11 +67,11 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) auto collection_copy = collection->duplicate(); - for (const auto & ast : asts) + for (auto it = std::next(asts.begin()); it != asts.end(); ++it) { - auto value_override = getKeyValueFromAST(ast); - if (!value_override) - continue; + auto value_override = getKeyValueFromAST(*it); + if (!value_override && !(*it)->as()) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected key-value argument or function"); const auto & [key, value] = *value_override; collection_copy->setOrUpdate(key, toString(value)); diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index b91df97782d..2982e960f3f 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -12,7 +12,6 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } - namespace DB { diff --git a/tests/integration/test_postgresql_database_engine/test.py b/tests/integration/test_postgresql_database_engine/test.py index d07f62f8a80..de6c9ad2cf9 100644 --- a/tests/integration/test_postgresql_database_engine/test.py +++ b/tests/integration/test_postgresql_database_engine/test.py @@ -282,6 +282,7 @@ def test_predefined_connection_configuration(started_cluster): result = node1.query( "select create_table_query from system.tables where database ='postgres_database'" ) + print(f"kssenii: {result}") assert result.strip().endswith( "ENGINE = PostgreSQL(postgres1, table = \\'test_table\\')" ) @@ -372,6 +373,19 @@ def test_postgresql_fetch_tables(started_cluster): cursor.execute("DROP SCHEMA IF EXISTS test_schema CASCADE") +def test_datetime(started_cluster): + cursor = started_cluster.postgres_conn.cursor() + cursor.execute("drop table if exists test") + cursor.execute("create table test (u timestamp)") + + node1.query("drop database if exists pg") + node1.query("create database pg engine = PostgreSQL(postgres1)") + assert "DateTime64(6)" in node1.query("show create table pg.test") + node1.query("detach table pg.test") + node1.query("attach table pg.test") + assert "DateTime64(6)" in node1.query("show create table pg.test") + + if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index 7cc350e0be2..a3ebbe97451 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -693,19 +693,6 @@ def test_auto_close_connection(started_cluster): assert count == 2 -def test_datetime(started_cluster): - cursor = started_cluster.postgres_conn.cursor() - cursor.execute("drop table if exists test") - cursor.execute("create table test (u timestamp)") - - node1.query("drop database if exists pg") - node1.query("create database pg engine = PostgreSQL(postgres1)") - assert "DateTime64(6)" in node1.query("show create table pg.test") - node1.query("detach table pg.test") - node1.query("attach table pg.test") - assert "DateTime64(6)" in node1.query("show create table pg.test") - - if __name__ == "__main__": cluster.start() input("Cluster created, press any key to destroy...") From 7d7efcbccd43a9f690879ed5d0030abd9d1605e6 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Thu, 22 Dec 2022 18:03:40 +0100 Subject: [PATCH 21/88] Add the lambda to collect data for workflow_jobs --- tests/ci/workflow_jobs_lambda/app.py | 299 ++++++++++++++++++ .../build_and_deploy_archive.sh | 1 + .../ci/workflow_jobs_lambda/requirements.txt | 1 + 3 files changed, 301 insertions(+) create mode 100644 tests/ci/workflow_jobs_lambda/app.py create mode 120000 tests/ci/workflow_jobs_lambda/build_and_deploy_archive.sh create mode 100644 tests/ci/workflow_jobs_lambda/requirements.txt diff --git a/tests/ci/workflow_jobs_lambda/app.py b/tests/ci/workflow_jobs_lambda/app.py new file mode 100644 index 00000000000..cdc4414c050 --- /dev/null +++ b/tests/ci/workflow_jobs_lambda/app.py @@ -0,0 +1,299 @@ +#!/usr/bin/env python + +""" +Lambda gets the workflow_job events, see +https://docs.github.com/en/developers/webhooks-and-events/webhooks/webhook-events-and-payloads#workflow_job + +Then it either posts it as is to the play.clickhouse.com, or anonymizes the sensitive +fields for private repositories +""" + +from base64 import b64decode +from dataclasses import dataclass +from typing import Any, List +import json +import logging +import time + +import boto3 # type: ignore +import requests # type: ignore + +logging.getLogger().setLevel(logging.INFO) + + +@dataclass +class WorkflowJob: + id: int + run_id: int + workflow_name: str + head_branch: str + run_url: str + run_attempt: int + node_id: str + head_sha: str + url: str + html_url: str + status: str + conclusion: str + started_at: str + completed_at: str + name: str + steps: int # just number of steps, we don't keep steps + check_run_url: str + labels: List[str] + runner_id: int + runner_name: str + runner_group_id: int + runner_group_name: str + repository: str + + def anonimyze_url(self, url: str) -> str: + return url.replace(self.repository, "ANONYMIZED_REPO") + + def anonimyze(self): + anm = "ANONYMIZED" + self.workflow_name = anm + self.head_branch = anm + self.run_url = self.anonimyze_url(self.run_url) + self.node_id = anm + self.url = self.anonimyze_url(self.url) + self.html_url = self.anonimyze_url(self.html_url) + self.name = anm + self.check_run_url = self.anonimyze_url(self.check_run_url) + self.repository = anm + + def as_dict(self) -> dict: + return self.__dict__ + + +### VENDORING +def get_parameter_from_ssm(name, decrypt=True, client=None): + if not client: + client = boto3.client("ssm", region_name="us-east-1") + return client.get_parameter(Name=name, WithDecryption=decrypt)["Parameter"]["Value"] + + +class InsertException(Exception): + pass + + +class ClickHouseHelper: + def __init__(self, url=None): + if url is None: + url = get_parameter_from_ssm("clickhouse-test-stat-url") + + self.url = url + self.auth = { + "X-ClickHouse-User": get_parameter_from_ssm("clickhouse-test-stat-login"), + "X-ClickHouse-Key": get_parameter_from_ssm("clickhouse-test-stat-password"), + } + + @staticmethod + def _insert_json_str_info_impl(url, auth, db, table, json_str): + params = { + "database": db, + "query": f"INSERT INTO {table} FORMAT JSONEachRow", + "date_time_input_format": "best_effort", + "send_logs_level": "warning", + } + + for i in range(5): + try: + response = requests.post( + url, params=params, data=json_str, headers=auth + ) + except Exception as e: + error = f"Received exception while sending data to {url} on {i} attempt: {e}" + logging.warning(error) + continue + + logging.info("Response content '%s'", response.content) + + if response.ok: + break + + error = ( + "Cannot insert data into clickhouse at try " + + str(i) + + ": HTTP code " + + str(response.status_code) + + ": '" + + str(response.text) + + "'" + ) + + if response.status_code >= 500: + # A retriable error + time.sleep(1) + continue + + logging.info( + "Request headers '%s', body '%s'", + response.request.headers, + response.request.body, + ) + + raise InsertException(error) + else: + raise InsertException(error) + + def _insert_json_str_info(self, db, table, json_str): + self._insert_json_str_info_impl(self.url, self.auth, db, table, json_str) + + def insert_event_into(self, db, table, event, safe=True): + event_str = json.dumps(event) + try: + self._insert_json_str_info(db, table, event_str) + except InsertException as e: + logging.error( + "Exception happened during inserting data into clickhouse: %s", e + ) + if not safe: + raise + + def insert_events_into(self, db, table, events, safe=True): + jsons = [] + for event in events: + jsons.append(json.dumps(event)) + + try: + self._insert_json_str_info(db, table, ",".join(jsons)) + except InsertException as e: + logging.error( + "Exception happened during inserting data into clickhouse: %s", e + ) + if not safe: + raise + + def _select_and_get_json_each_row(self, db, query): + params = { + "database": db, + "query": query, + "default_format": "JSONEachRow", + } + for i in range(5): + response = None + try: + response = requests.get(self.url, params=params, headers=self.auth) + response.raise_for_status() + return response.text + except Exception as ex: + logging.warning("Cannot insert with exception %s", str(ex)) + if response: + logging.warning("Reponse text %s", response.text) + time.sleep(0.1 * i) + + raise Exception("Cannot fetch data from clickhouse") + + def select_json_each_row(self, db, query): + text = self._select_and_get_json_each_row(db, query) + result = [] + for line in text.split("\n"): + if line: + result.append(json.loads(line)) + return result + + +### VENDORING END + +clickhouse_client = ClickHouseHelper() + + +def send_event_workflow_job(workflow_job: WorkflowJob) -> None: + # SHOW CREATE TABLE default.workflow_jobs + # CREATE TABLE default.workflow_jobs + # UUID 'c035192a-8ccd-47a6-9db0-f28a9eee2fde' + # ( + # `id` UInt64, + # `run_id` UInt64, + # `workflow_name` LowCardinality(String), + # `head_branch` LowCardinality(String), + # `run_url` String, + # `run_attempt` UInt16, + # `node_id` String, + # `head_sha` String, + # `url` String, + # `html_url` String, + # `status` Enum8('queued' = 1, 'in_progress' = 2, 'completed' = 3, 'waiting' = 4), + # `conclusion` LowCardinality(String), + # `started_at` DateTime, + # `completed_at` DateTime, + # `name` LowCardinality(String), + # `steps` UInt16, + # `check_run_url` String, + # `labels` Array(LowCardinality(String)), + # `runner_id` UInt64, + # `runner_name` String, + # `runner_group_id` UInt64, + # `runner_group_name` LowCardinality(String), + # `repository` LowCardinality(String) + # ) + # ENGINE = ReplicatedMergeTree( + # '/clickhouse/tables/c035192a-8ccd-47a6-9db0-f28a9eee2fde/{shard}', '{replica}' + # ) + # PARTITION BY toStartOfMonth(started_at) + # ORDER BY (id, started_at) + # SETTINGS index_granularity = 8192 + global clickhouse_client + kwargs = { + "db": "default", + "table": "workflow_jobs", + "event": workflow_job.as_dict(), + "safe": False, + } + try: + clickhouse_client.insert_event_into(**kwargs) + except InsertException as ex: + logging.exception( + "Got an exception on insert, tryuing to update the client " + "credentials and repeat", + exc_info=ex, + ) + clickhouse_client = ClickHouseHelper() + clickhouse_client.insert_event_into(**kwargs) + + +def handler(event: dict, _: Any) -> dict: + if event["isBase64Encoded"]: + event_data = json.loads(b64decode(event["body"])) + else: + event_data = json.loads(event["body"]) + + repo = event_data["repository"] + wf_job = event_data["workflow_job"] + workflow_job = WorkflowJob( + wf_job["id"], + wf_job["run_id"], + wf_job["workflow_name"] or "", # nullable + wf_job["head_branch"], + wf_job["run_url"], + wf_job["run_attempt"], + wf_job["node_id"], + wf_job["head_sha"], + wf_job["url"], + wf_job["html_url"], + wf_job["status"], + wf_job["conclusion"] or "", # nullable + wf_job["started_at"], + wf_job["completed_at"] or "1970-01-01T00:00:00", # nullable date + wf_job["name"], + len(wf_job["steps"]), + wf_job["check_run_url"], + wf_job["labels"], + wf_job["runner_id"] or 0, # nullable + wf_job["runner_name"] or "", # nullable + wf_job["runner_group_id"] or 0, # nullable + wf_job["runner_group_name"] or "", # nullable + repo["full_name"], + ) + if repo["private"]: + workflow_job.anonimyze() + + logging.info("Got the next event: %s", workflow_job) + send_event_workflow_job(workflow_job) + + return { + "statusCode": 200, + "headers": {"Content-Type": "application/json"}, + "body": '{"status": "OK"}', + } diff --git a/tests/ci/workflow_jobs_lambda/build_and_deploy_archive.sh b/tests/ci/workflow_jobs_lambda/build_and_deploy_archive.sh new file mode 120000 index 00000000000..96ba3fa024e --- /dev/null +++ b/tests/ci/workflow_jobs_lambda/build_and_deploy_archive.sh @@ -0,0 +1 @@ +../team_keys_lambda/build_and_deploy_archive.sh \ No newline at end of file diff --git a/tests/ci/workflow_jobs_lambda/requirements.txt b/tests/ci/workflow_jobs_lambda/requirements.txt new file mode 100644 index 00000000000..f2293605cf1 --- /dev/null +++ b/tests/ci/workflow_jobs_lambda/requirements.txt @@ -0,0 +1 @@ +requests From e9829b0d46537fce9c350f5f6e665e4fc943f311 Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Fri, 23 Dec 2022 12:05:05 +0100 Subject: [PATCH 22/88] Change the scheme to have rows ordered by updated_at --- tests/ci/workflow_jobs_lambda/app.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/tests/ci/workflow_jobs_lambda/app.py b/tests/ci/workflow_jobs_lambda/app.py index cdc4414c050..f6589576806 100644 --- a/tests/ci/workflow_jobs_lambda/app.py +++ b/tests/ci/workflow_jobs_lambda/app.py @@ -200,9 +200,8 @@ clickhouse_client = ClickHouseHelper() def send_event_workflow_job(workflow_job: WorkflowJob) -> None: - # SHOW CREATE TABLE default.workflow_jobs - # CREATE TABLE default.workflow_jobs - # UUID 'c035192a-8ccd-47a6-9db0-f28a9eee2fde' + # # SHOW CREATE TABLE default.workflow_jobs + # CREATE TABLE default.workflow_jobs UUID 'c0351924-8ccd-47a6-9db0-e28a9eee2fdf' # ( # `id` UInt64, # `run_id` UInt64, @@ -226,13 +225,12 @@ def send_event_workflow_job(workflow_job: WorkflowJob) -> None: # `runner_name` String, # `runner_group_id` UInt64, # `runner_group_name` LowCardinality(String), - # `repository` LowCardinality(String) - # ) - # ENGINE = ReplicatedMergeTree( - # '/clickhouse/tables/c035192a-8ccd-47a6-9db0-f28a9eee2fde/{shard}', '{replica}' + # `repository` LowCardinality(String), + # `updated_at` DateTime DEFAULT now() # ) + # ENGINE = ReplicatedMergeTree('/clickhouse/tables/c0351924-8ccd-47a6-9db0-e28a9eee2fdf/{shard}', '{replica}') # PARTITION BY toStartOfMonth(started_at) - # ORDER BY (id, started_at) + # ORDER BY (id, updated_at) # SETTINGS index_granularity = 8192 global clickhouse_client kwargs = { From e481c0bae57ea3af7425d4f85684b403bbe56a0a Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Fri, 23 Dec 2022 18:23:01 +0000 Subject: [PATCH 23/88] Cleanup code --- src/AggregateFunctions/AggregateFunctionMap.h | 8 +- .../AggregateFunctionOrFill.h | 4 +- .../AggregateFunctionSumMap.h | 5 +- src/AggregateFunctions/IAggregateFunction.h | 6 +- src/Analyzer/FunctionNode.cpp | 13 ++- src/Analyzer/FunctionNode.h | 105 +++++------------- .../Passes/CustomizeFunctionsPass.cpp | 2 +- .../Passes/FunctionToSubcolumnsPass.cpp | 2 +- src/Analyzer/Passes/FuseFunctionsPass.cpp | 4 +- src/Analyzer/Passes/IfChainToMultiIfPass.cpp | 2 +- .../Passes/IfTransformStringsToEnumPass.cpp | 8 +- src/Analyzer/Passes/MultiIfToIfPass.cpp | 2 +- src/Analyzer/Passes/QueryAnalysisPass.cpp | 4 +- src/Analyzer/Passes/SumIfToCountIfPass.cpp | 2 +- .../UniqInjectiveFunctionsEliminationPass.cpp | 1 - src/Analyzer/QueryTreePassManager.cpp | 2 +- src/Core/IResolvedFunction.h | 3 + src/Planner/Planner.cpp | 2 +- 18 files changed, 59 insertions(+), 116 deletions(-) diff --git a/src/AggregateFunctions/AggregateFunctionMap.h b/src/AggregateFunctions/AggregateFunctionMap.h index dc19bf3f71c..91530698bf4 100644 --- a/src/AggregateFunctions/AggregateFunctionMap.h +++ b/src/AggregateFunctions/AggregateFunctionMap.h @@ -116,13 +116,9 @@ public: static DataTypePtr getKeyType(const DataTypes & types, const AggregateFunctionPtr & nested) { - if (types.empty()) + if (types.size() != 1) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Aggregate function {}Map requires at least one argument", nested->getName()); - - if (types.size() > 1) - throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, - "Aggregate function {}Map requires only one map argument", nested->getName()); + "Aggregate function {}Map requires one map argument, but {} found", nested->getName(), types.size()); const auto * map_type = checkAndGetDataType(types[0].get()); if (!map_type) diff --git a/src/AggregateFunctions/AggregateFunctionOrFill.h b/src/AggregateFunctions/AggregateFunctionOrFill.h index eeec630be9a..4679e086506 100644 --- a/src/AggregateFunctions/AggregateFunctionOrFill.h +++ b/src/AggregateFunctions/AggregateFunctionOrFill.h @@ -36,8 +36,8 @@ public: AggregateFunctionOrFill(AggregateFunctionPtr nested_function_, const DataTypes & arguments, const Array & params) : IAggregateFunctionHelper{arguments, params, createResultType(nested_function_->getResultType())} , nested_function{nested_function_} - , size_of_data {nested_function->sizeOfData()} - , inner_nullable {nested_function->getResultType()->isNullable()} + , size_of_data{nested_function->sizeOfData()} + , inner_nullable{nested_function->getResultType()->isNullable()} { // nothing } diff --git a/src/AggregateFunctions/AggregateFunctionSumMap.h b/src/AggregateFunctions/AggregateFunctionSumMap.h index ee6b1525741..310491ff05c 100644 --- a/src/AggregateFunctions/AggregateFunctionSumMap.h +++ b/src/AggregateFunctions/AggregateFunctionSumMap.h @@ -428,10 +428,7 @@ public: } bool keepKey(const T & key) const { return static_cast(*this).keepKey(key); } - String getName() const override { return getNameImpl(); } - -private: - static String getNameImpl() { return Derived::getNameImpl(); } + String getName() const override { return Derived::getNameImpl(); } }; template diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index 1df5b67b155..517449f7465 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -65,9 +65,9 @@ class IAggregateFunction : public std::enable_shared_from_this(); } -ColumnsWithTypeAndName FunctionNode::getArgumentTypes() const +ColumnsWithTypeAndName FunctionNode::getArgumentColumns() const { - ColumnsWithTypeAndName argument_types; - for (const auto & arg : getArguments().getNodes()) + const auto & arguments = getArguments().getNodes(); + ColumnsWithTypeAndName argument_columns; + argument_columns.reserve(arguments.size()); + + for (const auto & arg : arguments) { ColumnWithTypeAndName argument; argument.type = arg->getResultType(); if (auto * constant = arg->as()) argument.column = argument.type->createColumnConst(1, constant->getValue()); - argument_types.push_back(argument); + argument_columns.push_back(argument); } - return argument_types; + return argument_columns; } void FunctionNode::resolveAsFunction(FunctionBasePtr function_value) diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index 501d439e55e..4fe8880ce5d 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -1,12 +1,14 @@ #pragma once #include -#include +#include +#include #include #include -#include -#include #include +#include +#include +#include namespace DB { @@ -19,12 +21,6 @@ namespace ErrorCodes class IFunctionOverloadResolver; using FunctionOverloadResolverPtr = std::shared_ptr; -class IFunctionBase; -using FunctionBasePtr = std::shared_ptr; - -class IAggregateFunction; -using AggregateFunctionPtr = std::shared_ptr; - /** Function node represents function in query tree. * Function syntax: function_name(parameter_1, ...)(argument_1, ...). * If function does not have parameters its syntax is function_name(argument_1, ...). @@ -63,66 +59,36 @@ public: explicit FunctionNode(String function_name_); /// Get function name - const String & getFunctionName() const - { - return function_name; - } + const String & getFunctionName() const { return function_name; } /// Get parameters - const ListNode & getParameters() const - { - return children[parameters_child_index]->as(); - } + const ListNode & getParameters() const { return children[parameters_child_index]->as(); } /// Get parameters - ListNode & getParameters() - { - return children[parameters_child_index]->as(); - } + ListNode & getParameters() { return children[parameters_child_index]->as(); } /// Get parameters node - const QueryTreeNodePtr & getParametersNode() const - { - return children[parameters_child_index]; - } + const QueryTreeNodePtr & getParametersNode() const { return children[parameters_child_index]; } /// Get parameters node - QueryTreeNodePtr & getParametersNode() - { - return children[parameters_child_index]; - } + QueryTreeNodePtr & getParametersNode() { return children[parameters_child_index]; } /// Get arguments - const ListNode & getArguments() const - { - return children[arguments_child_index]->as(); - } + const ListNode & getArguments() const { return children[arguments_child_index]->as(); } /// Get arguments - ListNode & getArguments() - { - return children[arguments_child_index]->as(); - } + ListNode & getArguments() { return children[arguments_child_index]->as(); } /// Get arguments node - const QueryTreeNodePtr & getArgumentsNode() const - { - return children[arguments_child_index]; - } + const QueryTreeNodePtr & getArgumentsNode() const { return children[arguments_child_index]; } /// Get arguments node - QueryTreeNodePtr & getArgumentsNode() - { - return children[arguments_child_index]; - } + QueryTreeNodePtr & getArgumentsNode() { return children[arguments_child_index]; } - ColumnsWithTypeAndName getArgumentTypes() const; + ColumnsWithTypeAndName getArgumentColumns() const; /// Returns true if function node has window, false otherwise - bool hasWindow() const - { - return children[window_child_index] != nullptr; - } + bool hasWindow() const { return children[window_child_index] != nullptr; } /** Get window node. * Valid only for window function node. @@ -130,18 +96,12 @@ public: * 1. It can be identifier node if window function is defined as expr OVER window_name. * 2. It can be window node if window function is defined as expr OVER (window_name ...). */ - const QueryTreeNodePtr & getWindowNode() const - { - return children[window_child_index]; - } + const QueryTreeNodePtr & getWindowNode() const { return children[window_child_index]; } /** Get window node. * Valid only for window function node. */ - QueryTreeNodePtr & getWindowNode() - { - return children[window_child_index]; - } + QueryTreeNodePtr & getWindowNode() { return children[window_child_index]; } /** Get non aggregate function. * If function is not resolved nullptr returned. @@ -150,7 +110,7 @@ public: { if (kind != FunctionKind::ORDINARY) return {}; - return std::reinterpret_pointer_cast(function); + return std::static_pointer_cast(function); } /** Get aggregate function. @@ -161,32 +121,20 @@ public: { if (kind == FunctionKind::UNKNOWN || kind == FunctionKind::ORDINARY) return {}; - return std::reinterpret_pointer_cast(function); + return std::static_pointer_cast(function); } /// Is function node resolved - bool isResolved() const - { - return function != nullptr; - } + bool isResolved() const { return function != nullptr; } /// Is function node window function - bool isWindowFunction() const - { - return hasWindow(); - } + bool isWindowFunction() const { return hasWindow(); } /// Is function node aggregate function - bool isAggregateFunction() const - { - return kind == FunctionKind::AGGREGATE; - } + bool isAggregateFunction() const { return kind == FunctionKind::AGGREGATE; } /// Is function node ordinary function - bool isOrdinaryFunction() const - { - return kind == FunctionKind::ORDINARY; - } + bool isOrdinaryFunction() const { return kind == FunctionKind::ORDINARY; } /** Resolve function node as non aggregate function. * It is important that function name is updated with resolved function name. @@ -208,10 +156,7 @@ public: */ void resolveAsWindowFunction(AggregateFunctionPtr window_function_value); - QueryTreeNodeType getNodeType() const override - { - return QueryTreeNodeType::FUNCTION; - } + QueryTreeNodeType getNodeType() const override { return QueryTreeNodeType::FUNCTION; } DataTypePtr getResultType() const override { diff --git a/src/Analyzer/Passes/CustomizeFunctionsPass.cpp b/src/Analyzer/Passes/CustomizeFunctionsPass.cpp index 7eb4a040970..407614dbe50 100644 --- a/src/Analyzer/Passes/CustomizeFunctionsPass.cpp +++ b/src/Analyzer/Passes/CustomizeFunctionsPass.cpp @@ -155,7 +155,7 @@ public: inline void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const { auto function = FunctionFactory::instance().get(function_name, context); - function_node.resolveAsFunction(function->build(function_node.getArgumentTypes())); + function_node.resolveAsFunction(function->build(function_node.getArgumentColumns())); } private: diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp index 0c5a450135f..aff932e39db 100644 --- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp +++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp @@ -193,7 +193,7 @@ private: inline void resolveOrdinaryFunctionNode(FunctionNode & function_node, const String & function_name) const { auto function = FunctionFactory::instance().get(function_name, context); - function_node.resolveAsFunction(function->build(function_node.getArgumentTypes())); + function_node.resolveAsFunction(function->build(function_node.getArgumentColumns())); } ContextPtr & context; diff --git a/src/Analyzer/Passes/FuseFunctionsPass.cpp b/src/Analyzer/Passes/FuseFunctionsPass.cpp index f354a7b1ec3..c73e1048dc2 100644 --- a/src/Analyzer/Passes/FuseFunctionsPass.cpp +++ b/src/Analyzer/Passes/FuseFunctionsPass.cpp @@ -65,7 +65,7 @@ QueryTreeNodePtr createResolvedFunction(const ContextPtr & context, const String auto function = FunctionFactory::instance().get(name, context); function_node->getArguments().getNodes() = std::move(arguments); - function_node->resolveAsFunction(function->build(function_node->getArgumentTypes())); + function_node->resolveAsFunction(function->build(function_node->getArgumentColumns())); return function_node; } @@ -88,7 +88,7 @@ FunctionNodePtr createResolvedAggregateFunction(const String & name, const Query { argument->getResultType() }, parameters, properties); - function_node->resolveAsAggregateFunction(aggregate_function); + function_node->resolveAsAggregateFunction(std::move(aggregate_function)); return function_node; } diff --git a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp index 020edfe4820..077ba331ead 100644 --- a/src/Analyzer/Passes/IfChainToMultiIfPass.cpp +++ b/src/Analyzer/Passes/IfChainToMultiIfPass.cpp @@ -56,7 +56,7 @@ public: auto multi_if_function = std::make_shared("multiIf"); multi_if_function->getArguments().getNodes() = std::move(multi_if_arguments); - multi_if_function->resolveAsFunction(multi_if_function_ptr->build(multi_if_function->getArgumentTypes())); + multi_if_function->resolveAsFunction(multi_if_function_ptr->build(multi_if_function->getArgumentColumns())); node = std::move(multi_if_function); } diff --git a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp index 776fe63c803..dd1c211a053 100644 --- a/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp +++ b/src/Analyzer/Passes/IfTransformStringsToEnumPass.cpp @@ -52,7 +52,7 @@ QueryTreeNodePtr createCastFunction(QueryTreeNodePtr from, DataTypePtr result_ty auto function_node = std::make_shared("_CAST"); function_node->getArguments().getNodes() = std::move(arguments); - function_node->resolveAsFunction(cast_function->build(function_node->getArgumentTypes())); + function_node->resolveAsFunction(cast_function->build(function_node->getArgumentColumns())); return function_node; } @@ -71,7 +71,7 @@ void changeIfArguments( auto if_resolver = FunctionFactory::instance().get("if", context); - if_node.resolveAsFunction(if_resolver->build(if_node.getArgumentTypes())); + if_node.resolveAsFunction(if_resolver->build(if_node.getArgumentColumns())); } /// transform(value, array_from, array_to, default_value) will be transformed to transform(value, array_from, _CAST(array_to, Array(Enum...)), _CAST(default_value, Enum...)) @@ -93,7 +93,7 @@ void changeTransformArguments( auto transform_resolver = FunctionFactory::instance().get("transform", context); - transform_node.resolveAsFunction(transform_resolver->build(transform_node.getArgumentTypes())); + transform_node.resolveAsFunction(transform_resolver->build(transform_node.getArgumentColumns())); } void wrapIntoToString(FunctionNode & function_node, QueryTreeNodePtr arg, ContextPtr context) @@ -102,7 +102,7 @@ void wrapIntoToString(FunctionNode & function_node, QueryTreeNodePtr arg, Contex QueryTreeNodes arguments{ std::move(arg) }; function_node.getArguments().getNodes() = std::move(arguments); - function_node.resolveAsFunction(to_string_function->build(function_node.getArgumentTypes())); + function_node.resolveAsFunction(to_string_function->build(function_node.getArgumentColumns())); assert(isString(function_node.getResultType())); } diff --git a/src/Analyzer/Passes/MultiIfToIfPass.cpp b/src/Analyzer/Passes/MultiIfToIfPass.cpp index 7e13675bf98..9eb2a4da817 100644 --- a/src/Analyzer/Passes/MultiIfToIfPass.cpp +++ b/src/Analyzer/Passes/MultiIfToIfPass.cpp @@ -27,7 +27,7 @@ public: return; auto result_type = function_node->getResultType(); - function_node->resolveAsFunction(if_function_ptr->build(function_node->getArgumentTypes())); + function_node->resolveAsFunction(if_function_ptr->build(function_node->getArgumentColumns())); } private: diff --git a/src/Analyzer/Passes/QueryAnalysisPass.cpp b/src/Analyzer/Passes/QueryAnalysisPass.cpp index c8a86a4f036..9da5fb7ffa6 100644 --- a/src/Analyzer/Passes/QueryAnalysisPass.cpp +++ b/src/Analyzer/Passes/QueryAnalysisPass.cpp @@ -4327,7 +4327,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi AggregateFunctionProperties properties; auto aggregate_function = AggregateFunctionFactory::instance().get(function_name, argument_types, parameters, properties); - function_node.resolveAsWindowFunction(aggregate_function); + function_node.resolveAsWindowFunction(std::move(aggregate_function)); bool window_node_is_identifier = function_node.getWindowNode()->getNodeType() == QueryTreeNodeType::IDENTIFIER; ProjectionName window_projection_name = resolveWindow(function_node.getWindowNode(), scope); @@ -4386,7 +4386,7 @@ ProjectionNames QueryAnalyzer::resolveFunction(QueryTreeNodePtr & node, Identifi AggregateFunctionProperties properties; auto aggregate_function = AggregateFunctionFactory::instance().get(function_name, argument_types, parameters, properties); - function_node.resolveAsAggregateFunction(aggregate_function); + function_node.resolveAsAggregateFunction(std::move(aggregate_function)); return result_projection_names; } diff --git a/src/Analyzer/Passes/SumIfToCountIfPass.cpp b/src/Analyzer/Passes/SumIfToCountIfPass.cpp index 1496d539d27..bc121807da7 100644 --- a/src/Analyzer/Passes/SumIfToCountIfPass.cpp +++ b/src/Analyzer/Passes/SumIfToCountIfPass.cpp @@ -121,7 +121,7 @@ public: auto & not_function_arguments = not_function->getArguments().getNodes(); not_function_arguments.push_back(std::move(nested_if_function_arguments_nodes[0])); - not_function->resolveAsFunction(FunctionFactory::instance().get("not", context)->build(not_function->getArgumentTypes())); + not_function->resolveAsFunction(FunctionFactory::instance().get("not", context)->build(not_function->getArgumentColumns())); function_node_arguments_nodes[0] = std::move(not_function); function_node_arguments_nodes.resize(1); diff --git a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp index 37bad70da57..3ce5ec4a24c 100644 --- a/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp +++ b/src/Analyzer/Passes/UniqInjectiveFunctionsEliminationPass.cpp @@ -75,7 +75,6 @@ public: function_node->getAggregateFunction()->getParameters(), properties); - auto function_result_type = function_node->getResultType(); function_node->resolveAsAggregateFunction(std::move(aggregate_function)); } }; diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 06a1fec4698..1fa6297c8c0 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -59,7 +59,7 @@ class ValidationChecker : public InDepthQueryTreeVisitor if (!function->isResolved()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Function {} is not resolved after running {} pass", - function->dumpTree(), pass_name); + function->toAST()->dumpTree(), pass_name); } public: diff --git a/src/Core/IResolvedFunction.h b/src/Core/IResolvedFunction.h index 64c69f597c7..d472d2ce734 100644 --- a/src/Core/IResolvedFunction.h +++ b/src/Core/IResolvedFunction.h @@ -12,6 +12,9 @@ using DataTypes = std::vector; struct Array; +/* Generic class for all functions. + * Represents interface for function signature. + */ class IResolvedFunction { public: diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 6b2de30722c..a0e8c4687c6 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -350,7 +350,7 @@ void Planner::buildQueryPlanIfNeeded() auto function_node = std::make_shared("and"); auto and_function = FunctionFactory::instance().get("and", query_context); function_node->getArguments().getNodes() = {query_node.getPrewhere(), query_node.getWhere()}; - function_node->resolveAsFunction(and_function->build(function_node->getArgumentTypes())); + function_node->resolveAsFunction(and_function->build(function_node->getArgumentColumns())); query_node.getWhere() = std::move(function_node); query_node.getPrewhere() = {}; } From 81c0db3b67bf39b4d71184803394a6acc4ba6ee7 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 29 Dec 2022 15:47:43 +0000 Subject: [PATCH 24/88] Reuse FunctionBase for IndexHint --- src/Storages/MergeTree/KeyCondition.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 5d0c3fc3cad..2340bdd99b2 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -613,7 +613,7 @@ static const ActionsDAG::Node & cloneASTWithInversionPushDown( } } - const auto & func = inverted_dag.addFunction(FunctionFactory::instance().get(node.function_base->getName(), context), children, ""); + const auto & func = inverted_dag.addFunction(node.function_base, children, ""); to_inverted[&node] = &func; return func; } From 93cf734b57265e61e3fac744c9dfecc154642890 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Thu, 29 Dec 2022 15:49:51 +0000 Subject: [PATCH 25/88] Refactor ActionsDAG::addFunction --- src/Interpreters/ActionsDAG.cpp | 138 +++++++++++--------------------- src/Interpreters/ActionsDAG.h | 10 +++ 2 files changed, 58 insertions(+), 90 deletions(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 3b4d2dd1dd4..bceffcc5708 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -161,85 +161,37 @@ const ActionsDAG::Node & ActionsDAG::addFunction( NodeRawConstPtrs children, std::string result_name) { - size_t num_arguments = children.size(); + auto [arguments, all_const] = getFunctionArguments(children); - Node node; - node.type = ActionType::FUNCTION; - node.children = std::move(children); - - bool all_const = true; - ColumnsWithTypeAndName arguments(num_arguments); - - for (size_t i = 0; i < num_arguments; ++i) - { - const auto & child = *node.children[i]; - - ColumnWithTypeAndName argument; - argument.column = child.column; - argument.type = child.result_type; - argument.name = child.result_name; - - if (!argument.column || !isColumnConst(*argument.column)) - all_const = false; - - arguments[i] = std::move(argument); - } - - node.function_base = function->build(arguments); - node.result_type = node.function_base->getResultType(); - node.function = node.function_base->prepare(arguments); - node.is_deterministic = node.function_base->isDeterministic(); - - /// If all arguments are constants, and function is suitable to be executed in 'prepare' stage - execute function. - if (node.function_base->isSuitableForConstantFolding()) - { - ColumnPtr column; - - if (all_const) - { - size_t num_rows = arguments.empty() ? 0 : arguments.front().column->size(); - column = node.function->execute(arguments, node.result_type, num_rows, true); - } - else - { - column = node.function_base->getConstantResultForNonConstArguments(arguments, node.result_type); - } - - /// If the result is not a constant, just in case, we will consider the result as unknown. - if (column && isColumnConst(*column)) - { - /// All constant (literal) columns in block are added with size 1. - /// But if there was no columns in block before executing a function, the result has size 0. - /// Change the size to 1. - - if (column->empty()) - column = column->cloneResized(1); - - node.column = std::move(column); - } - } - - if (result_name.empty()) - { - result_name = function->getName() + "("; - for (size_t i = 0; i < num_arguments; ++i) - { - if (i) - result_name += ", "; - result_name += node.children[i]->result_name; - } - result_name += ")"; - } - - node.result_name = std::move(result_name); - - return addNode(std::move(node)); + return addFunctionImpl( + function->build(arguments), + std::move(children), + std::move(arguments), + std::move(result_name), + all_const); } const ActionsDAG::Node & ActionsDAG::addFunction( const FunctionBasePtr & function_base, NodeRawConstPtrs children, std::string result_name) +{ + auto [arguments, all_const] = getFunctionArguments(children); + + return addFunctionImpl( + function_base, + std::move(children), + std::move(arguments), + std::move(result_name), + all_const); +} + +const ActionsDAG::Node & ActionsDAG::addFunctionImpl( + const FunctionBasePtr & function_base, + NodeRawConstPtrs children, + ColumnsWithTypeAndName arguments, + std::string result_name, + bool all_const) { size_t num_arguments = children.size(); @@ -247,24 +199,6 @@ const ActionsDAG::Node & ActionsDAG::addFunction( node.type = ActionType::FUNCTION; node.children = std::move(children); - bool all_const = true; - ColumnsWithTypeAndName arguments(num_arguments); - - for (size_t i = 0; i < num_arguments; ++i) - { - const auto & child = *node.children[i]; - - ColumnWithTypeAndName argument; - argument.column = child.column; - argument.type = child.result_type; - argument.name = child.result_name; - - if (!argument.column || !isColumnConst(*argument.column)) - all_const = false; - - arguments[i] = std::move(argument); - } - node.function_base = function_base; node.result_type = node.function_base->getResultType(); node.function = node.function_base->prepare(arguments); @@ -316,6 +250,30 @@ const ActionsDAG::Node & ActionsDAG::addFunction( return addNode(std::move(node)); } +std::pair ActionsDAG::getFunctionArguments(const NodeRawConstPtrs & children) +{ + size_t num_arguments = children.size(); + + bool all_const = true; + ColumnsWithTypeAndName arguments(num_arguments); + + for (size_t i = 0; i < num_arguments; ++i) + { + const auto & child = *children[i]; + + ColumnWithTypeAndName argument; + argument.column = child.column; + argument.type = child.result_type; + argument.name = child.result_name; + + if (!argument.column || !isColumnConst(*argument.column)) + all_const = false; + + arguments[i] = std::move(argument); + } + return { std::move(arguments), all_const }; +} + const ActionsDAG::Node & ActionsDAG::findInOutputs(const std::string & name) const { if (const auto * node = tryFindInOutputs(name)) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index f574757abac..8d1ea470277 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -1,5 +1,6 @@ #pragma once +#include #include #include #include @@ -350,6 +351,15 @@ private: Node & addNode(Node node); + const Node & addFunctionImpl( + const FunctionBasePtr & function_base, + NodeRawConstPtrs children, + ColumnsWithTypeAndName arguments, + std::string result_name, + bool all_const); + + static std::pair getFunctionArguments(const NodeRawConstPtrs & children); + #if USE_EMBEDDED_COMPILER void compileFunctions(size_t min_count_to_compile_expression, const std::unordered_set & lazy_executed_nodes = {}); #endif From e52ef6cbcd0192aec00d2e0f4a9071d242283821 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 29 Dec 2022 21:35:20 +0100 Subject: [PATCH 26/88] Update NamedCollectionsHelpers.h --- src/Storages/NamedCollectionsHelpers.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Storages/NamedCollectionsHelpers.h b/src/Storages/NamedCollectionsHelpers.h index 2982e960f3f..a726af82a95 100644 --- a/src/Storages/NamedCollectionsHelpers.h +++ b/src/Storages/NamedCollectionsHelpers.h @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -50,8 +51,8 @@ void validateNamedCollection( { throw Exception( ErrorCodes::BAD_ARGUMENTS, - "Unexpected key `{}` in named collection. Required keys: {}, optional keys: {}", - key, fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); + "Unexpected key {} in named collection. Required keys: {}, optional keys: {}", + backQuoteIfNeed(key), fmt::join(required_keys, ", "), fmt::join(optional_keys, ", ")); } } From c846720f3d7aa0f065dd7336fa9dc71684a1c444 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Fri, 30 Dec 2022 16:12:18 +0100 Subject: [PATCH 27/88] Fix for qualified asterisks with alias table name and column transformer Implementation: * Removed check that qualified asterisks should have only 1 child. It can have more than 1 children when the rest of the children are column transformers. Added check for the same. * Updated ExtractAsterisksMatcher data to store table name and table alias name as both can be referenced in the query. Testing: * Added a test 02518_qualified_asterisks_alias_table_name.sql --- .../JoinToSubqueryTransformVisitor.cpp | 18 ++++++++++--- ...ified_asterisks_alias_table_name.reference | 1 + ...8_qualified_asterisks_alias_table_name.sql | 25 +++++++++++++++++++ 3 files changed, 40 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/02518_qualified_asterisks_alias_table_name.reference create mode 100644 tests/queries/0_stateless/02518_qualified_asterisks_alias_table_name.sql diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index 15d12de527d..db842ee3446 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -60,6 +60,7 @@ public: struct Data { std::unordered_map table_columns; + std::unordered_map table_name_alias; std::vector tables_order; std::shared_ptr new_select_expression_list; @@ -71,6 +72,7 @@ public: String table_name = table.table.getQualifiedNamePrefix(false); NamesAndTypesList columns = table.columns; tables_order.push_back(table_name); + table_name_alias.emplace(table.table.table /* table_name */, table_name /* alias_name */); table_columns.emplace(std::move(table_name), std::move(columns)); } } @@ -85,7 +87,14 @@ public: ASTs & columns, ShouldAddColumnPredicate should_add_column_predicate = [](const String &) { return true; }) { - auto it = table_columns.find(table_name); + String name = table_name; + auto table_name_it = table_name_alias.find(table_name); + if (table_name_it != table_name_alias.end()) + { + name = table_name_it->second; + } + + auto it = table_columns.find(name); if (it == table_columns.end()) throw Exception("Unknown qualified identifier: " + table_name, ErrorCodes::UNKNOWN_IDENTIFIER); @@ -146,8 +155,6 @@ private: { has_asterisks = true; - if (child->children.size() != 1) - throw Exception("Logical error: qualified asterisk must have exactly one child", ErrorCodes::LOGICAL_ERROR); auto & identifier = child->children[0]->as(); data.addTableColumns(identifier.name(), columns); @@ -155,7 +162,10 @@ private: // QualifiedAsterisk's transformers start to appear at child 1 for (const auto * it = qualified_asterisk->children.begin() + 1; it != qualified_asterisk->children.end(); ++it) { - IASTColumnsTransformer::transform(*it, columns); + if (it->get()->as() || it->get()->as() || it->get()->as()) + IASTColumnsTransformer::transform(*it, columns); + else + throw Exception("Logical error: qualified asterisk must only have children of IASTColumnsTransformer type", ErrorCodes::LOGICAL_ERROR); } } else if (const auto * columns_list_matcher = child->as()) diff --git a/tests/queries/0_stateless/02518_qualified_asterisks_alias_table_name.reference b/tests/queries/0_stateless/02518_qualified_asterisks_alias_table_name.reference new file mode 100644 index 00000000000..b250b4e6616 --- /dev/null +++ b/tests/queries/0_stateless/02518_qualified_asterisks_alias_table_name.reference @@ -0,0 +1 @@ +1 a 1 b 1 c diff --git a/tests/queries/0_stateless/02518_qualified_asterisks_alias_table_name.sql b/tests/queries/0_stateless/02518_qualified_asterisks_alias_table_name.sql new file mode 100644 index 00000000000..808fa58a993 --- /dev/null +++ b/tests/queries/0_stateless/02518_qualified_asterisks_alias_table_name.sql @@ -0,0 +1,25 @@ +DROP TABLE IF EXISTS test_table_join_1; +CREATE TABLE test_table_join_1 (id UInt64, value String) ENGINE = TinyLog; + +DROP TABLE IF EXISTS test_table_join_2; +CREATE TABLE test_table_join_2 (id UInt64, value String) ENGINE = TinyLog; + +DROP TABLE IF EXISTS test_table_join_3; +CREATE TABLE test_table_join_3 (id UInt64, value String ) ENGINE = TinyLog; + +INSERT INTO test_table_join_1 VALUES (1, 'a'); +INSERT INTO test_table_join_2 VALUES (1, 'b'); +INSERT INTO test_table_join_3 VALUES (1, 'c'); + + +SELECT + test_table_join_1.* APPLY toString, + test_table_join_2.* APPLY toString, + test_table_join_3.* APPLY toString +FROM test_table_join_1 AS t1 + INNER JOIN test_table_join_2 AS t2 ON t1.id = t2.id + INNER JOIN test_table_join_3 AS t3 ON t2.id = t3.id; + +DROP TABLE test_table_join_1; +DROP TABLE test_table_join_2; +DROP TABLE test_table_join_3; \ No newline at end of file From 60a34b2f369505fbe0ee107b6e0008cf1abf005e Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Mon, 2 Jan 2023 09:40:49 +0100 Subject: [PATCH 28/88] Fixed issue with table alias for failing tests - Fix for qualified asterisks with alias table name and column transformer --- .../JoinToSubqueryTransformVisitor.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp index db842ee3446..a57b8d2354b 100644 --- a/src/Interpreters/JoinToSubqueryTransformVisitor.cpp +++ b/src/Interpreters/JoinToSubqueryTransformVisitor.cpp @@ -88,15 +88,20 @@ public: ShouldAddColumnPredicate should_add_column_predicate = [](const String &) { return true; }) { String name = table_name; - auto table_name_it = table_name_alias.find(table_name); - if (table_name_it != table_name_alias.end()) - { - name = table_name_it->second; - } - auto it = table_columns.find(name); if (it == table_columns.end()) - throw Exception("Unknown qualified identifier: " + table_name, ErrorCodes::UNKNOWN_IDENTIFIER); + { + auto table_name_it = table_name_alias.find(table_name); + if (table_name_it != table_name_alias.end()) + { + name = table_name_it->second; + it = table_columns.find(table_name_it->second); + if (it == table_columns.end()) + throw Exception("Unknown qualified identifier: " + table_name, ErrorCodes::UNKNOWN_IDENTIFIER); + } + else + throw Exception("Unknown qualified identifier: " + table_name, ErrorCodes::UNKNOWN_IDENTIFIER); + } for (const auto & column : it->second) { From 4c03ccf99bef3f5c8c4fe63952943d331feedc29 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Mon, 2 Jan 2023 12:06:18 +0000 Subject: [PATCH 29/88] Sanity check that the SelectAST isn't corrupt --- src/Parsers/ASTSelectQuery.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Parsers/ASTSelectQuery.cpp b/src/Parsers/ASTSelectQuery.cpp index 46355e16575..a287bd13481 100644 --- a/src/Parsers/ASTSelectQuery.cpp +++ b/src/Parsers/ASTSelectQuery.cpp @@ -113,8 +113,10 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F if (group_by_with_cube) s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << (s.one_line ? "" : " ") << "WITH CUBE" << (s.hilite ? hilite_none : ""); - if (groupBy() && group_by_with_grouping_sets) + if (group_by_with_grouping_sets) { + if (!groupBy()) /// sanity check, issue 43049 + throw Exception(ErrorCodes::LOGICAL_ERROR, "Corrupt AST"); auto nested_frame = frame; nested_frame.surround_each_list_element_with_parens = true; nested_frame.expression_list_prepend_whitespace = false; From b40f6c0b10b10f39b4504ab09629f8eb36357ba1 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 2 Jan 2023 15:04:14 +0000 Subject: [PATCH 30/88] Pre-pull docker image and fail check completely if it failed --- tests/integration/ci-runner.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index e640b3a1463..5a4807b2252 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -160,6 +160,24 @@ def get_test_times(output): return result +def pre_pull_images(): + + images = ["clickhouse/integration-test"] + for image in images: + for i in range(5): + logging.info("Pulling %s image before running tests. Attempt %s", image, i) + try: + subprocess.check_output( + "docker pull %s".format(image), + shell=True, + ) + except subprocess.CalledProcessError as err: + logging.info("docker pull failed: " + str(err)) + continue + logging.error("Pulling %s failed for 5 attempts. Will fail the worker.", image) + exit(1) + + def clear_ip_tables_and_restart_daemons(): logging.info( "Dump iptables after run %s", From 631c55e379dfd5558d8e42a9ecb57e4fb51ccd0e Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 2 Jan 2023 16:45:50 +0000 Subject: [PATCH 31/88] Fix compose files to work without correct environment --- .../compose/docker_compose_clickhouse.yml | 5 +++++ .../runner/compose/docker_compose_hdfs.yml | 6 +++--- .../runner/compose/docker_compose_kafka.yml | 4 ++-- .../docker_compose_kerberized_hdfs.yml | 4 ++-- .../docker_compose_kerberized_kafka.yml | 4 ++-- .../runner/compose/docker_compose_meili.yml | 4 ++-- .../runner/compose/docker_compose_minio.yml | 15 +++++++------ .../runner/compose/docker_compose_mongo.yml | 4 ++-- .../compose/docker_compose_mongo_secure.yml | 4 ++-- .../runner/compose/docker_compose_mysql.yml | 2 +- ...mpose_mysql_5_7_for_materialized_mysql.yml | 21 ------------------- .../compose/docker_compose_mysql_8_0.yml | 2 +- .../compose/docker_compose_mysql_cluster.yml | 6 +++--- .../runner/compose/docker_compose_nats.yml | 4 ++-- .../compose/docker_compose_postgres.yml | 2 +- .../docker_compose_postgres_cluster.yml | 6 +++--- .../compose/docker_compose_rabbitmq.yml | 2 +- .../runner/compose/docker_compose_redis.yml | 2 +- 18 files changed, 40 insertions(+), 57 deletions(-) create mode 100644 docker/test/integration/runner/compose/docker_compose_clickhouse.yml delete mode 100644 docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml diff --git a/docker/test/integration/runner/compose/docker_compose_clickhouse.yml b/docker/test/integration/runner/compose/docker_compose_clickhouse.yml new file mode 100644 index 00000000000..fdd124ede91 --- /dev/null +++ b/docker/test/integration/runner/compose/docker_compose_clickhouse.yml @@ -0,0 +1,5 @@ +version: '2.3' +# Used to pre-pull images with docker-compose +services: + clickhouse1: + image: clickhouse/integration-test diff --git a/docker/test/integration/runner/compose/docker_compose_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_hdfs.yml index 4c1f2c08f05..f83eb93fea7 100644 --- a/docker/test/integration/runner/compose/docker_compose_hdfs.yml +++ b/docker/test/integration/runner/compose/docker_compose_hdfs.yml @@ -5,10 +5,10 @@ services: hostname: hdfs1 restart: always expose: - - ${HDFS_NAME_PORT} - - ${HDFS_DATA_PORT} + - ${HDFS_NAME_PORT:-50070} + - ${HDFS_DATA_PORT:-50075} entrypoint: /etc/bootstrap.sh -d volumes: - type: ${HDFS_FS:-tmpfs} source: ${HDFS_LOGS:-} - target: /usr/local/hadoop/logs \ No newline at end of file + target: /usr/local/hadoop/logs diff --git a/docker/test/integration/runner/compose/docker_compose_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kafka.yml index 4552e6f0696..7e34f4c114d 100644 --- a/docker/test/integration/runner/compose/docker_compose_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kafka.yml @@ -15,7 +15,7 @@ services: image: confluentinc/cp-kafka:5.2.0 hostname: kafka1 ports: - - ${KAFKA_EXTERNAL_PORT}:${KAFKA_EXTERNAL_PORT} + - ${KAFKA_EXTERNAL_PORT:-8081}:${KAFKA_EXTERNAL_PORT:-8081} environment: KAFKA_ADVERTISED_LISTENERS: INSIDE://localhost:${KAFKA_EXTERNAL_PORT},OUTSIDE://kafka1:19092 KAFKA_ADVERTISED_HOST_NAME: kafka1 @@ -35,7 +35,7 @@ services: image: confluentinc/cp-schema-registry:5.2.0 hostname: schema-registry ports: - - ${SCHEMA_REGISTRY_EXTERNAL_PORT}:${SCHEMA_REGISTRY_INTERNAL_PORT} + - ${SCHEMA_REGISTRY_EXTERNAL_PORT:-12313}:${SCHEMA_REGISTRY_INTERNAL_PORT:-12313} environment: SCHEMA_REGISTRY_HOST_NAME: schema-registry SCHEMA_REGISTRY_KAFKASTORE_SECURITY_PROTOCOL: PLAINTEXT diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml index e1b4d393169..1160192696d 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_hdfs.yml @@ -15,8 +15,8 @@ services: source: ${KERBERIZED_HDFS_LOGS:-} target: /var/log/hadoop-hdfs expose: - - ${KERBERIZED_HDFS_NAME_PORT} - - ${KERBERIZED_HDFS_DATA_PORT} + - ${KERBERIZED_HDFS_NAME_PORT:-50070} + - ${KERBERIZED_HDFS_DATA_PORT:-1006} depends_on: - hdfskerberos entrypoint: /etc/bootstrap.sh -d diff --git a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml index d57e4e4d5be..86e920ff573 100644 --- a/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml +++ b/docker/test/integration/runner/compose/docker_compose_kerberized_kafka.yml @@ -23,7 +23,7 @@ services: # restart: always hostname: kerberized_kafka1 ports: - - ${KERBERIZED_KAFKA_EXTERNAL_PORT}:${KERBERIZED_KAFKA_EXTERNAL_PORT} + - ${KERBERIZED_KAFKA_EXTERNAL_PORT:-19092}:${KERBERIZED_KAFKA_EXTERNAL_PORT:-19092} environment: KAFKA_LISTENERS: OUTSIDE://:19092,UNSECURED_OUTSIDE://:19093,UNSECURED_INSIDE://0.0.0.0:${KERBERIZED_KAFKA_EXTERNAL_PORT} KAFKA_ADVERTISED_LISTENERS: OUTSIDE://kerberized_kafka1:19092,UNSECURED_OUTSIDE://kerberized_kafka1:19093,UNSECURED_INSIDE://localhost:${KERBERIZED_KAFKA_EXTERNAL_PORT} @@ -41,7 +41,7 @@ services: KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 KAFKA_OPTS: "-Djava.security.auth.login.config=/etc/kafka/secrets/broker_jaas.conf -Djava.security.krb5.conf=/etc/kafka/secrets/krb.conf -Dsun.security.krb5.debug=true" volumes: - - ${KERBERIZED_KAFKA_DIR}/secrets:/etc/kafka/secrets + - ${KERBERIZED_KAFKA_DIR:-}/secrets:/etc/kafka/secrets - /dev/urandom:/dev/random depends_on: - kafka_kerberized_zookeeper diff --git a/docker/test/integration/runner/compose/docker_compose_meili.yml b/docker/test/integration/runner/compose/docker_compose_meili.yml index dc58643d26f..c734c43b4c6 100644 --- a/docker/test/integration/runner/compose/docker_compose_meili.yml +++ b/docker/test/integration/runner/compose/docker_compose_meili.yml @@ -4,13 +4,13 @@ services: image: getmeili/meilisearch:v0.27.0 restart: always ports: - - ${MEILI_EXTERNAL_PORT}:${MEILI_INTERNAL_PORT} + - ${MEILI_EXTERNAL_PORT:-7700}:${MEILI_INTERNAL_PORT:-7700} meili_secure: image: getmeili/meilisearch:v0.27.0 restart: always ports: - - ${MEILI_SECURE_EXTERNAL_PORT}:${MEILI_SECURE_INTERNAL_PORT} + - ${MEILI_SECURE_EXTERNAL_PORT:-7700}:${MEILI_SECURE_INTERNAL_PORT:-7700} environment: MEILI_MASTER_KEY: "password" diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/docker/test/integration/runner/compose/docker_compose_minio.yml index 6e8c826b234..eb9b73c0f29 100644 --- a/docker/test/integration/runner/compose/docker_compose_minio.yml +++ b/docker/test/integration/runner/compose/docker_compose_minio.yml @@ -2,19 +2,18 @@ version: '2.3' services: minio1: - # Newer version of minio results in such errors: - # "AWSErrorMarshaller: Encountered AWSError 'InternalError': We encountered an internal error, please try again" - image: minio/minio:RELEASE.2021-09-23T04-46-24Z + image: minio/minio:RELEASE.2022-12-12T19-27-27Z volumes: - - data1-1:/data1 + - ${MINIO_DATA_DIR}:/data1 - ${MINIO_CERTS_DIR:-}:/certs + - ${MINIO_LOGS_DIR:-}:/logs expose: - - ${MINIO_PORT} + - ${MINIO_PORT:-9001} environment: - MINIO_ACCESS_KEY: minio - MINIO_SECRET_KEY: minio123 + MINIO_ROOT_USER: minio + MINIO_ROOT_PASSWORD: minio123 MINIO_PROMETHEUS_AUTH_TYPE: public - command: server --address :9001 --certs-dir /certs /data1-1 + command: server --address :9001 --certs-dir /certs /data1 depends_on: - proxy1 - proxy2 diff --git a/docker/test/integration/runner/compose/docker_compose_mongo.yml b/docker/test/integration/runner/compose/docker_compose_mongo.yml index 80190b0a293..9a6eae6ca8c 100644 --- a/docker/test/integration/runner/compose/docker_compose_mongo.yml +++ b/docker/test/integration/runner/compose/docker_compose_mongo.yml @@ -7,11 +7,11 @@ services: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: clickhouse ports: - - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT} + - ${MONGO_EXTERNAL_PORT:-27017}:${MONGO_INTERNAL_PORT:-27017} command: --profile=2 --verbose mongo2: image: mongo:5.0 restart: always ports: - - ${MONGO_NO_CRED_EXTERNAL_PORT}:${MONGO_NO_CRED_INTERNAL_PORT} + - ${MONGO_NO_CRED_EXTERNAL_PORT:-27017}:${MONGO_NO_CRED_INTERNAL_PORT:-27017} diff --git a/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml b/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml index 5d283cfc343..193e5d26568 100644 --- a/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml +++ b/docker/test/integration/runner/compose/docker_compose_mongo_secure.yml @@ -7,7 +7,7 @@ services: MONGO_INITDB_ROOT_USERNAME: root MONGO_INITDB_ROOT_PASSWORD: clickhouse volumes: - - ${MONGO_CONFIG_PATH}:/mongo/ + - ${MONGO_CONFIG_PATH:-}:/mongo/ ports: - - ${MONGO_EXTERNAL_PORT}:${MONGO_INTERNAL_PORT} + - ${MONGO_EXTERNAL_PORT:-27017}:${MONGO_INTERNAL_PORT:-27017} command: --config /mongo/mongo_secure.conf --profile=2 --verbose diff --git a/docker/test/integration/runner/compose/docker_compose_mysql.yml b/docker/test/integration/runner/compose/docker_compose_mysql.yml index c9b45af6563..6b98a372bd0 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql.yml @@ -8,7 +8,7 @@ services: MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST} DATADIR: /mysql/ expose: - - ${MYSQL_PORT} + - ${MYSQL_PORT:-3306} command: --server_id=100 --log-bin='mysql-bin-1.log' --default-time-zone='+3:00' diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml b/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml deleted file mode 100644 index ba693fd9fb4..00000000000 --- a/docker/test/integration/runner/compose/docker_compose_mysql_5_7_for_materialized_mysql.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: '2.3' -services: - mysql1: - image: mysql:5.7 - restart: 'no' - environment: - MYSQL_ROOT_PASSWORD: clickhouse - ports: - - 3308:3306 - command: --server_id=100 --log-bin='mysql-bin-1.log' - --default-time-zone='+3:00' - --gtid-mode="ON" - --enforce-gtid-consistency - --log-error-verbosity=3 - --log-error=/var/log/mysqld/error.log - --general-log=ON - --general-log-file=/var/log/mysqld/general.log - volumes: - - type: ${MYSQL_LOGS_FS:-tmpfs} - source: ${MYSQL_LOGS:-} - target: /var/log/mysqld/ \ No newline at end of file diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml b/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml index e13076c4e2e..d5fb5a53aaf 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_8_0.yml @@ -8,7 +8,7 @@ services: MYSQL_ROOT_HOST: ${MYSQL_ROOT_HOST} DATADIR: /mysql/ expose: - - ${MYSQL8_PORT} + - ${MYSQL8_PORT:-3306} command: --server_id=100 --log-bin='mysql-bin-1.log' --default_authentication_plugin='mysql_native_password' --default-time-zone='+3:00' --gtid-mode="ON" diff --git a/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml b/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml index 6ced7536812..8e145a3b408 100644 --- a/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml +++ b/docker/test/integration/runner/compose/docker_compose_mysql_cluster.yml @@ -8,7 +8,7 @@ services: MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST} DATADIR: /mysql/ expose: - - ${MYSQL_CLUSTER_PORT} + - ${MYSQL_CLUSTER_PORT:-3306} command: --server_id=100 --log-bin='mysql-bin-2.log' --default-time-zone='+3:00' @@ -30,7 +30,7 @@ services: MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST} DATADIR: /mysql/ expose: - - ${MYSQL_CLUSTER_PORT} + - ${MYSQL_CLUSTER_PORT:-3306} command: --server_id=100 --log-bin='mysql-bin-3.log' --default-time-zone='+3:00' @@ -52,7 +52,7 @@ services: MYSQL_ROOT_HOST: ${MYSQL_CLUSTER_ROOT_HOST} DATADIR: /mysql/ expose: - - ${MYSQL_CLUSTER_PORT} + - ${MYSQL_CLUSTER_PORT:-3306} command: --server_id=100 --log-bin='mysql-bin-4.log' --default-time-zone='+3:00' diff --git a/docker/test/integration/runner/compose/docker_compose_nats.yml b/docker/test/integration/runner/compose/docker_compose_nats.yml index 2122f0f639f..b17ac62fa93 100644 --- a/docker/test/integration/runner/compose/docker_compose_nats.yml +++ b/docker/test/integration/runner/compose/docker_compose_nats.yml @@ -3,9 +3,9 @@ services: nats1: image: nats ports: - - "${NATS_EXTERNAL_PORT}:${NATS_INTERNAL_PORT}" + - "${NATS_EXTERNAL_PORT:-4444}:${NATS_INTERNAL_PORT:-4444}" command: "-p 4444 --user click --pass house --tls --tlscert=/etc/certs/server-cert.pem --tlskey=/etc/certs/server-key.pem" volumes: - type: bind - source: "${NATS_CERT_DIR}/nats" + source: "${NATS_CERT_DIR:-}/nats" target: /etc/certs diff --git a/docker/test/integration/runner/compose/docker_compose_postgres.yml b/docker/test/integration/runner/compose/docker_compose_postgres.yml index 15ea548e218..1fb6b7a1410 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgres.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgres.yml @@ -5,7 +5,7 @@ services: command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=2", "-c", "logging_collector=on", "-c", "log_directory=/postgres/logs", "-c", "log_filename=postgresql.log", "-c", "log_statement=all", "-c", "max_connections=200"] restart: always expose: - - ${POSTGRES_PORT} + - ${POSTGRES_PORT:-5432} healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 10s diff --git a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml index 94b941b74da..5af13ca3e0f 100644 --- a/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml +++ b/docker/test/integration/runner/compose/docker_compose_postgres_cluster.yml @@ -9,7 +9,7 @@ services: POSTGRES_PASSWORD: mysecretpassword PGDATA: /postgres/data expose: - - ${POSTGRES_PORT} + - ${POSTGRES_PORT:-5432} volumes: - type: ${POSTGRES_LOGS_FS:-tmpfs} source: ${POSTGRES2_DIR:-} @@ -23,7 +23,7 @@ services: POSTGRES_PASSWORD: mysecretpassword PGDATA: /postgres/data expose: - - ${POSTGRES_PORT} + - ${POSTGRES_PORT:-5432} volumes: - type: ${POSTGRES_LOGS_FS:-tmpfs} source: ${POSTGRES3_DIR:-} @@ -37,7 +37,7 @@ services: POSTGRES_PASSWORD: mysecretpassword PGDATA: /postgres/data expose: - - ${POSTGRES_PORT} + - ${POSTGRES_PORT:-5432} volumes: - type: ${POSTGRES_LOGS_FS:-tmpfs} source: ${POSTGRES4_DIR:-} diff --git a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml index 539c065e03b..0190516728c 100644 --- a/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml +++ b/docker/test/integration/runner/compose/docker_compose_rabbitmq.yml @@ -5,7 +5,7 @@ services: image: rabbitmq:3.8-management-alpine hostname: rabbitmq1 expose: - - ${RABBITMQ_PORT} + - ${RABBITMQ_PORT:-5672} environment: RABBITMQ_DEFAULT_USER: "root" RABBITMQ_DEFAULT_PASS: "clickhouse" diff --git a/docker/test/integration/runner/compose/docker_compose_redis.yml b/docker/test/integration/runner/compose/docker_compose_redis.yml index 1bf67a6c44e..e2aa836ae46 100644 --- a/docker/test/integration/runner/compose/docker_compose_redis.yml +++ b/docker/test/integration/runner/compose/docker_compose_redis.yml @@ -4,5 +4,5 @@ services: image: redis restart: always ports: - - ${REDIS_EXTERNAL_PORT}:${REDIS_INTERNAL_PORT} + - ${REDIS_EXTERNAL_PORT:-6379}:${REDIS_INTERNAL_PORT:-6379} command: redis-server --requirepass "clickhouse" --databases 32 From 9ab5cf7d625d926a85dac81f07d024d131ee17b1 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 2 Jan 2023 16:46:24 +0000 Subject: [PATCH 32/88] Download inside runner --- tests/integration/ci-runner.py | 45 ++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 18 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 5a4807b2252..d6770e572df 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -160,24 +160,6 @@ def get_test_times(output): return result -def pre_pull_images(): - - images = ["clickhouse/integration-test"] - for image in images: - for i in range(5): - logging.info("Pulling %s image before running tests. Attempt %s", image, i) - try: - subprocess.check_output( - "docker pull %s".format(image), - shell=True, - ) - except subprocess.CalledProcessError as err: - logging.info("docker pull failed: " + str(err)) - continue - logging.error("Pulling %s failed for 5 attempts. Will fail the worker.", image) - exit(1) - - def clear_ip_tables_and_restart_daemons(): logging.info( "Dump iptables after run %s", @@ -311,6 +293,30 @@ class ClickhouseIntegrationTestsRunner: "clickhouse/postgresql-java-client", ] + def _pre_pull_images(): + cmd = ( + "cd {repo_path}/tests/integration && " + "timeout -s 9 1h ./runner {runner_opts} {image_cmd} {command} ".format( + repo_path=repo_path, + runner_opts=self._get_runner_opts(), + image_cmd=image_cmd, + command=r""" find /compose -name 'docker_compose_*.yml' -exec docker-compose -f '{}' pull \; """ + ) + ) + + for i in range(5): + logging.info("Pulling images before running tests. Attempt %s", image, i) + try: + subprocess.check_output( + cmd, + shell=True, + ) + except subprocess.CalledProcessError as err: + logging.info("docker-compose pull failed: " + str(err)) + return + logging.error("Pulling %s failed for 5 attempts. Will fail the worker.", image) + exit(1) + def _can_run_with(self, path, opt): with open(path, "r") as script: for line in script: @@ -1000,6 +1006,9 @@ if __name__ == "__main__": params = json.loads(open(params_path, "r").read()) runner = ClickhouseIntegrationTestsRunner(result_path, params) + logging.info("Pulling images") + runner._pre_pull_images() + logging.info("Running tests") # Avoid overlaps with previous runs From fb9e3632042423bf688bb6b512d1d624eb5a37cf Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 2 Jan 2023 17:06:07 +0000 Subject: [PATCH 33/88] remove unnecessary change --- docker/test/integration/runner/compose/docker_compose_minio.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/docker/test/integration/runner/compose/docker_compose_minio.yml index eb9b73c0f29..45f9ef3c454 100644 --- a/docker/test/integration/runner/compose/docker_compose_minio.yml +++ b/docker/test/integration/runner/compose/docker_compose_minio.yml @@ -6,7 +6,6 @@ services: volumes: - ${MINIO_DATA_DIR}:/data1 - ${MINIO_CERTS_DIR:-}:/certs - - ${MINIO_LOGS_DIR:-}:/logs expose: - ${MINIO_PORT:-9001} environment: From 3251d9278c48567cc70cb48bef84af2d80133358 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Mon, 2 Jan 2023 17:44:04 +0000 Subject: [PATCH 34/88] black --- tests/integration/ci-runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index d6770e572df..0fbdcbce6a6 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -300,7 +300,7 @@ class ClickhouseIntegrationTestsRunner: repo_path=repo_path, runner_opts=self._get_runner_opts(), image_cmd=image_cmd, - command=r""" find /compose -name 'docker_compose_*.yml' -exec docker-compose -f '{}' pull \; """ + command=r""" find /compose -name 'docker_compose_*.yml' -exec docker-compose -f '{}' pull \; """, ) ) From 3deada0c29379c5d411f824d7aeea77d7b9b5469 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Jan 2023 16:05:20 +0000 Subject: [PATCH 35/88] fix --- tests/integration/ci-runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 0fbdcbce6a6..1e330c9b76e 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -293,7 +293,7 @@ class ClickhouseIntegrationTestsRunner: "clickhouse/postgresql-java-client", ] - def _pre_pull_images(): + def _pre_pull_images(self): cmd = ( "cd {repo_path}/tests/integration && " "timeout -s 9 1h ./runner {runner_opts} {image_cmd} {command} ".format( From 6386e839536979e30aa06b6f4bf94dbe921fa458 Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Tue, 3 Jan 2023 16:50:06 +0000 Subject: [PATCH 36/88] Fixes after review --- src/Analyzer/FunctionNode.cpp | 2 +- src/Analyzer/FunctionNode.h | 2 +- src/Analyzer/QueryTreePassManager.cpp | 2 +- src/Interpreters/ActionsDAG.cpp | 53 +++++++++++++++------------ src/Interpreters/ActionsDAG.h | 2 - 5 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/Analyzer/FunctionNode.cpp b/src/Analyzer/FunctionNode.cpp index e9970babdec..84a1d5025e3 100644 --- a/src/Analyzer/FunctionNode.cpp +++ b/src/Analyzer/FunctionNode.cpp @@ -43,7 +43,7 @@ ColumnsWithTypeAndName FunctionNode::getArgumentColumns() const argument.type = arg->getResultType(); if (auto * constant = arg->as()) argument.column = argument.type->createColumnConst(1, constant->getValue()); - argument_columns.push_back(argument); + argument_columns.push_back(std::move(argument)); } return argument_columns; } diff --git a/src/Analyzer/FunctionNode.h b/src/Analyzer/FunctionNode.h index 4fe8880ce5d..49e66ba32c1 100644 --- a/src/Analyzer/FunctionNode.h +++ b/src/Analyzer/FunctionNode.h @@ -5,9 +5,9 @@ #include #include #include +#include #include #include -#include #include namespace DB diff --git a/src/Analyzer/QueryTreePassManager.cpp b/src/Analyzer/QueryTreePassManager.cpp index 1fa6297c8c0..4148d42ee23 100644 --- a/src/Analyzer/QueryTreePassManager.cpp +++ b/src/Analyzer/QueryTreePassManager.cpp @@ -59,7 +59,7 @@ class ValidationChecker : public InDepthQueryTreeVisitor if (!function->isResolved()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Function {} is not resolved after running {} pass", - function->toAST()->dumpTree(), pass_name); + function->toAST()->formatForErrorMessage(), pass_name); } public: diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index bceffcc5708..6e935898ddd 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -33,6 +33,35 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } +namespace +{ + +std::pair getFunctionArguments(const ActionsDAG::NodeRawConstPtrs & children) +{ + size_t num_arguments = children.size(); + + bool all_const = true; + ColumnsWithTypeAndName arguments(num_arguments); + + for (size_t i = 0; i < num_arguments; ++i) + { + const auto & child = *children[i]; + + ColumnWithTypeAndName argument; + argument.column = child.column; + argument.type = child.result_type; + argument.name = child.result_name; + + if (!argument.column || !isColumnConst(*argument.column)) + all_const = false; + + arguments[i] = std::move(argument); + } + return { std::move(arguments), all_const }; +} + +} + void ActionsDAG::Node::toTree(JSONBuilder::JSONMap & map) const { map.add("Node Type", magic_enum::enum_name(type)); @@ -250,30 +279,6 @@ const ActionsDAG::Node & ActionsDAG::addFunctionImpl( return addNode(std::move(node)); } -std::pair ActionsDAG::getFunctionArguments(const NodeRawConstPtrs & children) -{ - size_t num_arguments = children.size(); - - bool all_const = true; - ColumnsWithTypeAndName arguments(num_arguments); - - for (size_t i = 0; i < num_arguments; ++i) - { - const auto & child = *children[i]; - - ColumnWithTypeAndName argument; - argument.column = child.column; - argument.type = child.result_type; - argument.name = child.result_name; - - if (!argument.column || !isColumnConst(*argument.column)) - all_const = false; - - arguments[i] = std::move(argument); - } - return { std::move(arguments), all_const }; -} - const ActionsDAG::Node & ActionsDAG::findInOutputs(const std::string & name) const { if (const auto * node = tryFindInOutputs(name)) diff --git a/src/Interpreters/ActionsDAG.h b/src/Interpreters/ActionsDAG.h index 8d1ea470277..a26694e00f5 100644 --- a/src/Interpreters/ActionsDAG.h +++ b/src/Interpreters/ActionsDAG.h @@ -358,8 +358,6 @@ private: std::string result_name, bool all_const); - static std::pair getFunctionArguments(const NodeRawConstPtrs & children); - #if USE_EMBEDDED_COMPILER void compileFunctions(size_t min_count_to_compile_expression, const std::unordered_set & lazy_executed_nodes = {}); #endif From da26f62a9b2c30eb660dcbc4237f4c760cb34c98 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 3 Jan 2023 18:19:51 +0000 Subject: [PATCH 37/88] Fix right offset for reading LowCardinality dictionary from remote fs in case if right mark was in the middle of compressed block. --- src/Formats/MarkInCompressedFile.h | 5 + .../MergeTree/MergeTreeReaderStream.cpp | 111 ++++++++++-------- .../MergeTree/MergeTreeReaderStream.h | 2 +- .../test.py | 21 ++++ 4 files changed, 89 insertions(+), 50 deletions(-) diff --git a/src/Formats/MarkInCompressedFile.h b/src/Formats/MarkInCompressedFile.h index 1cd545e1a03..287d3f7909d 100644 --- a/src/Formats/MarkInCompressedFile.h +++ b/src/Formats/MarkInCompressedFile.h @@ -28,6 +28,11 @@ struct MarkInCompressedFile return !(*this == rhs); } + auto asTuple() const + { + return std::make_tuple(offset_in_compressed_file, offset_in_decompressed_block); + } + String toString() const { return "(" + DB::toString(offset_in_compressed_file) + "," + DB::toString(offset_in_decompressed_block) + ")"; diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 47f8b0f6008..0ac454ec9ab 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -135,7 +135,7 @@ void MergeTreeReaderStream::init() } -size_t MergeTreeReaderStream::getRightOffset(size_t right_mark_non_included) +size_t MergeTreeReaderStream::getRightOffset(size_t right_mark) { /// NOTE: if we are reading the whole file, then right_mark == marks_count /// and we will use max_read_buffer_size for buffer size, thus avoiding the need to load marks. @@ -144,70 +144,83 @@ size_t MergeTreeReaderStream::getRightOffset(size_t right_mark_non_included) if (marks_count == 0) return 0; - assert(right_mark_non_included <= marks_count); + assert(right_mark <= marks_count); - size_t result_right_offset; - if (0 < right_mark_non_included && right_mark_non_included < marks_count) + if (0 < right_mark && right_mark < marks_count) { /// Find the right border of the last mark we need to read. /// To do that let's find the upper bound of the offset of the last /// included mark. - /// In LowCardinality dictionary and in values of Sparse columns - /// several consecutive marks can point to the same offset. - /// - /// Example: - /// Mark 186, points to [2003111, 0] - /// Mark 187, points to [2003111, 0] - /// Mark 188, points to [2003111, 0] <--- for example need to read until 188 - /// Mark 189, points to [2003111, 0] <--- not suitable, because have same offset - /// Mark 190, points to [2003111, 0] - /// Mark 191, points to [2003111, 0] - /// Mark 192, points to [2081424, 0] <--- what we are looking for - /// Mark 193, points to [2081424, 0] - /// Mark 194, points to [2081424, 0] + if (is_low_cardinality_dictionary) + { - /// Also, in some cases, when one granule is not-atomically written (which is possible at merges) - /// one granule may require reading of two dictionaries which starts from different marks. - /// The only correct way is to take offset from at least next different granule from the right one. - /// So, that's why we have to read one extra granule to the right, - /// while reading dictionary of LowCardinality. + /// In LowCardinality dictionary several consecutive marks can point to the same offset. + /// + /// Also, in some cases, when one granule is not-atomically written (which is possible at merges) + /// one granule may require reading of two dictionaries which starts from different marks. + /// The only correct way is to take offset from at least next different granule from the right one. + /// So, that's why we have to read one extra granule to the right, + /// while reading dictionary of LowCardinality. + /// + /// Example: + /// Mark 0, points to [0, 8] + /// Mark 1, points to [0, 8] + /// Mark 2, points to [0, 8] + /// Mark 3, points to [0, 8] + /// Mark 4, points to [42336, 2255] + /// Mark 5, points to [42336, 2255] <--- for example need to read until 5 + /// Mark 6, points to [42336, 2255] <--- not suitable, because have same offset + /// Mark 7, points to [84995, 7738] <--- next different mark + /// Mark 8, points to [84995, 7738] + /// Mark 9, points to [126531, 8637] <--- what we are looking for - /// If right_mark_non_included has non-zero offset in decompressed block, we have to - /// read its compressed block in a whole, because it may consist data from previous granule. + auto indices = collections::range(right_mark, marks_count); + auto next_different_mark = [&](auto lhs, auto rhs) + { + return marks_loader.getMark(lhs).asTuple() < marks_loader.getMark(rhs).asTuple(); + }; + auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_mark)); + + if (it == indices.end()) + return file_size; + + right_mark = *it; + } + + /// This is a good scenario. The compressed block is finished witin the right mark, + /// and previous mark was different. + if (marks_loader.getMark(right_mark).offset_in_decompressed_block == 0 + && marks_loader.getMark(right_mark) != marks_loader.getMark(right_mark - 1)) + return marks_loader.getMark(right_mark).offset_in_compressed_file; + + /// If right_mark has non-zero offset in decompressed block, we have to + /// read its compressed block in a whole, because it may consist of data from previous granule. /// /// For example: - /// Mark 10: (758287, 0) <--- right_mark_included - /// Mark 11: (908457, 53477) <--- right_mark_non_included - /// Mark 12: (1064746, 20742) <--- what we are looking for - /// Mark 13: (2009333, 40123) + /// Mark 6, points to [42336, 2255] + /// Mark 7, points to [84995, 7738] <--- right_mark + /// Mark 8, points to [84995, 7738] + /// Mark 9, points to [126531, 8637] <--- what we are looking for /// - /// Since mark 11 starts from offset in decompressed block 53477, - /// it has some data from mark 10 and we have to read - /// compressed block [908457; 1064746 in a whole. + /// Since mark 7 starts from offset in decompressed block 7738, + /// it has some data from mark 6 and we have to read + /// compressed block [84995; 126531 in a whole. - size_t right_mark_included = right_mark_non_included - 1; - if (is_low_cardinality_dictionary || marks_loader.getMark(right_mark_non_included).offset_in_decompressed_block != 0) - ++right_mark_included; - - auto indices = collections::range(right_mark_included, marks_count); - auto it = std::upper_bound(indices.begin(), indices.end(), right_mark_included, - [&](auto lhs, auto rhs) - { - return marks_loader.getMark(lhs).offset_in_compressed_file < marks_loader.getMark(rhs).offset_in_compressed_file; - }); + auto indices = collections::range(right_mark, marks_count); + auto next_different_compressed_offset = [&](auto lhs, auto rhs) + { + return marks_loader.getMark(lhs).offset_in_compressed_file < marks_loader.getMark(rhs).offset_in_compressed_file; + }; + auto it = std::upper_bound(indices.begin(), indices.end(), right_mark, std::move(next_different_compressed_offset)); if (it != indices.end()) - result_right_offset = marks_loader.getMark(*it).offset_in_compressed_file; - else - result_right_offset = file_size; + return marks_loader.getMark(*it).offset_in_compressed_file; } - else if (right_mark_non_included == 0) - result_right_offset = marks_loader.getMark(right_mark_non_included).offset_in_compressed_file; - else - result_right_offset = file_size; + else if (right_mark == 0) + return marks_loader.getMark(right_mark).offset_in_compressed_file; - return result_right_offset; + return file_size; } void MergeTreeReaderStream::seekToMark(size_t index) diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.h b/src/Storages/MergeTree/MergeTreeReaderStream.h index 83e314eef42..f3785e175df 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.h +++ b/src/Storages/MergeTree/MergeTreeReaderStream.h @@ -49,7 +49,7 @@ public: private: void init(); - size_t getRightOffset(size_t right_mark_non_included); + size_t getRightOffset(size_t right_mark); const MergeTreeReaderSettings settings; const ReadBufferFromFileBase::ProfileCallback profile_callback; diff --git a/tests/integration/test_s3_low_cardinality_right_border/test.py b/tests/integration/test_s3_low_cardinality_right_border/test.py index 14476719c5f..3f49dbb898b 100644 --- a/tests/integration/test_s3_low_cardinality_right_border/test.py +++ b/tests/integration/test_s3_low_cardinality_right_border/test.py @@ -132,3 +132,24 @@ def test_s3_right_border_2(started_cluster): node1.query("optimize table s3_low_cardinality final") res = node1.query("select * from s3_low_cardinality where key = 9000") assert res == "9000\t9000\n" + +def test_s3_right_border_3(started_cluster): + node1.query("drop table if exists s3_low_cardinality") + node1.query("create table s3_low_cardinality (x LowCardinality(String)) engine = MergeTree order by tuple() settings min_bytes_for_wide_part=0, storage_policy = 's3', max_compress_block_size=10000") + node1.query("insert into s3_low_cardinality select toString(number % 8000) || if(number < 8192 * 3, 'aaaaaaaaaaaaaaaa', if(number < 8192 * 6, 'bbbbbbbbbbbbbbbbbbbbbbbb', 'ccccccccccccccccccc')) from numbers(8192 * 9)") + # Marks are: + # Mark 0, points to 0, 8, has rows after 8192, decompressed size 0. + # Mark 1, points to 0, 8, has rows after 8192, decompressed size 0. + # Mark 2, points to 0, 8, has rows after 8192, decompressed size 0. + # Mark 3, points to 0, 8, has rows after 8192, decompressed size 0. + # Mark 4, points to 42336, 2255, has rows after 8192, decompressed size 0. + # Mark 5, points to 42336, 2255, has rows after 8192, decompressed size 0. + # Mark 6, points to 42336, 2255, has rows after 8192, decompressed size 0. + # Mark 7, points to 84995, 7738, has rows after 8192, decompressed size 0. + # Mark 8, points to 84995, 7738, has rows after 8192, decompressed size 0. + # Mark 9, points to 126531, 8637, has rows after 0, decompressed size 0. + + res = node1.query("select uniq(x) from s3_low_cardinality settings max_threads=2, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem=1, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem=1") + # Reading ranges [0, 5) and [5, 9) + + assert res == "23999\n" From d6c090b5309547dcfeecfc45aedcc1a8c6da0a75 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Jan 2023 18:29:09 +0000 Subject: [PATCH 38/88] Automatic style fix --- .../test_s3_low_cardinality_right_border/test.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_s3_low_cardinality_right_border/test.py b/tests/integration/test_s3_low_cardinality_right_border/test.py index 3f49dbb898b..e54e6783b3d 100644 --- a/tests/integration/test_s3_low_cardinality_right_border/test.py +++ b/tests/integration/test_s3_low_cardinality_right_border/test.py @@ -133,10 +133,15 @@ def test_s3_right_border_2(started_cluster): res = node1.query("select * from s3_low_cardinality where key = 9000") assert res == "9000\t9000\n" + def test_s3_right_border_3(started_cluster): node1.query("drop table if exists s3_low_cardinality") - node1.query("create table s3_low_cardinality (x LowCardinality(String)) engine = MergeTree order by tuple() settings min_bytes_for_wide_part=0, storage_policy = 's3', max_compress_block_size=10000") - node1.query("insert into s3_low_cardinality select toString(number % 8000) || if(number < 8192 * 3, 'aaaaaaaaaaaaaaaa', if(number < 8192 * 6, 'bbbbbbbbbbbbbbbbbbbbbbbb', 'ccccccccccccccccccc')) from numbers(8192 * 9)") + node1.query( + "create table s3_low_cardinality (x LowCardinality(String)) engine = MergeTree order by tuple() settings min_bytes_for_wide_part=0, storage_policy = 's3', max_compress_block_size=10000" + ) + node1.query( + "insert into s3_low_cardinality select toString(number % 8000) || if(number < 8192 * 3, 'aaaaaaaaaaaaaaaa', if(number < 8192 * 6, 'bbbbbbbbbbbbbbbbbbbbbbbb', 'ccccccccccccccccccc')) from numbers(8192 * 9)" + ) # Marks are: # Mark 0, points to 0, 8, has rows after 8192, decompressed size 0. # Mark 1, points to 0, 8, has rows after 8192, decompressed size 0. @@ -149,7 +154,9 @@ def test_s3_right_border_3(started_cluster): # Mark 8, points to 84995, 7738, has rows after 8192, decompressed size 0. # Mark 9, points to 126531, 8637, has rows after 0, decompressed size 0. - res = node1.query("select uniq(x) from s3_low_cardinality settings max_threads=2, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem=1, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem=1") + res = node1.query( + "select uniq(x) from s3_low_cardinality settings max_threads=2, merge_tree_min_rows_for_concurrent_read_for_remote_filesystem=1, merge_tree_min_bytes_for_concurrent_read_for_remote_filesystem=1" + ) # Reading ranges [0, 5) and [5, 9) assert res == "23999\n" From 5257ff627bd739582bf365d0c03fe0a9b11d07cb Mon Sep 17 00:00:00 2001 From: "Mikhail f. Shiryaev" Date: Mon, 2 Jan 2023 21:13:12 +0100 Subject: [PATCH 39/88] Optimize and fix zookeeper downloading and installation --- docker/test/integration/base/Dockerfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/docker/test/integration/base/Dockerfile b/docker/test/integration/base/Dockerfile index a2d86187a23..92dea21b1a3 100644 --- a/docker/test/integration/base/Dockerfile +++ b/docker/test/integration/base/Dockerfile @@ -57,14 +57,17 @@ RUN arch=${TARGETARCH:-amd64} \ # ZooKeeper is not started by default, but consumes some space in containers. # 777 perms used to allow anybody to start/stop ZooKeeper ENV ZOOKEEPER_VERSION='3.6.3' -RUN curl -O "https://dlcdn.apache.org/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz" -RUN tar -zxvf apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz && mv apache-zookeeper-${ZOOKEEPER_VERSION}-bin /opt/zookeeper && chmod -R 777 /opt/zookeeper && rm apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz -RUN echo $'tickTime=2500 \n\ +RUN curl "https://archive.apache.org/dist/zookeeper/zookeeper-${ZOOKEEPER_VERSION}/apache-zookeeper-${ZOOKEEPER_VERSION}-bin.tar.gz" | \ + tar -C opt -zxv && \ + mv /opt/apache-zookeeper-${ZOOKEEPER_VERSION}-bin /opt/zookeeper && \ + chmod -R 777 /opt/zookeeper && \ + echo $'tickTime=2500 \n\ tickTime=2500 \n\ dataDir=/zookeeper \n\ clientPort=2181 \n\ -maxClientCnxns=80' > /opt/zookeeper/conf/zoo.cfg -RUN mkdir /zookeeper && chmod -R 777 /zookeeper +maxClientCnxns=80' > /opt/zookeeper/conf/zoo.cfg && \ + mkdir /zookeeper && \ + chmod -R 777 /zookeeper ENV TZ=Etc/UTC RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone From c51f5e6fac5fcbfef1bb37ba3cbff89eb4c23fea Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Tue, 3 Jan 2023 18:41:37 +0000 Subject: [PATCH 40/88] Fix typo. --- src/Storages/MergeTree/MergeTreeReaderStream.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeReaderStream.cpp b/src/Storages/MergeTree/MergeTreeReaderStream.cpp index 0ac454ec9ab..cdca5aa1247 100644 --- a/src/Storages/MergeTree/MergeTreeReaderStream.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderStream.cpp @@ -188,7 +188,7 @@ size_t MergeTreeReaderStream::getRightOffset(size_t right_mark) right_mark = *it; } - /// This is a good scenario. The compressed block is finished witin the right mark, + /// This is a good scenario. The compressed block is finished within the right mark, /// and previous mark was different. if (marks_loader.getMark(right_mark).offset_in_decompressed_block == 0 && marks_loader.getMark(right_mark) != marks_loader.getMark(right_mark - 1)) From 683df2dea5de9989642677af8a5dca9787b002f6 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Jan 2023 18:56:24 +0000 Subject: [PATCH 41/88] fix --- tests/integration/ci-runner.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 1e330c9b76e..b94e6a56e2c 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -293,7 +293,9 @@ class ClickhouseIntegrationTestsRunner: "clickhouse/postgresql-java-client", ] - def _pre_pull_images(self): + def _pre_pull_images(self, repo_path): + image_cmd = self._get_runner_image_cmd(repo_path) + cmd = ( "cd {repo_path}/tests/integration && " "timeout -s 9 1h ./runner {runner_opts} {image_cmd} {command} ".format( @@ -1007,7 +1009,7 @@ if __name__ == "__main__": runner = ClickhouseIntegrationTestsRunner(result_path, params) logging.info("Pulling images") - runner._pre_pull_images() + runner._pre_pull_images(repo_path) logging.info("Running tests") From 14274d650bcbfbe0173d0717f729d88e1c9ed69e Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Jan 2023 19:30:14 +0000 Subject: [PATCH 42/88] Rewrite test_postgres_protocol test --- tests/integration/helpers/cluster.py | 87 +++++++- .../queries/query1.sql | 1 + .../queries/query2.sql | 1 + .../queries/query3.sql | 1 + .../queries/query4.sql | 1 + .../test_postgresql_protocol/test.py | 191 ++++++------------ 6 files changed, 142 insertions(+), 140 deletions(-) create mode 100644 tests/integration/test_postgresql_protocol/queries/query1.sql create mode 100644 tests/integration/test_postgresql_protocol/queries/query2.sql create mode 100644 tests/integration/test_postgresql_protocol/queries/query3.sql create mode 100644 tests/integration/test_postgresql_protocol/queries/query4.sql diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index 95d405266ae..eabbc489ba2 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -206,6 +206,13 @@ def check_kafka_is_available(kafka_id, kafka_port): p.communicate() return p.returncode == 0 +def check_postgresql_java_client_is_available(postgresql_java_client_id): + p = subprocess.Popen( + ("docker", "exec", "-i", postgresql_java_client_id, "java", "-version"), + stdout=subprocess.PIPE, + ) + p.communicate() + return p.returncode == 0 def check_rabbitmq_is_available(rabbitmq_id): p = subprocess.Popen( @@ -259,12 +266,8 @@ def enable_consistent_hash_plugin(rabbitmq_id): def get_instances_dir(name): instances_dir_name = "_instances" - worker_name = os.environ.get("PYTEST_XDIST_WORKER", "") run_id = os.environ.get("INTEGRATION_TESTS_RUN_ID", "") - if worker_name: - instances_dir_name += "_" + worker_name - if name: instances_dir_name += "_" + name @@ -386,6 +389,7 @@ class ClickHouseCluster: self.with_mysql_cluster = False self.with_postgres = False self.with_postgres_cluster = False + self.with_postgresql_java_client = False self.with_kafka = False self.with_kerberized_kafka = False self.with_rabbitmq = False @@ -404,9 +408,11 @@ class ClickHouseCluster: self.with_hive = False self.with_coredns = False + # available when with_minio == True self.with_minio = False self.minio_dir = os.path.join(self.instances_dir, "minio") self.minio_certs_dir = None # source for certificates + self.minio_data_dir = p.join(self.minio_dir, "data") self.minio_host = "minio1" self.minio_ip = None self.minio_bucket = "root" @@ -516,6 +522,11 @@ class ClickHouseCluster: self.postgres2_logs_dir = os.path.join(self.postgres_dir, "postgres2") self.postgres3_logs_dir = os.path.join(self.postgres_dir, "postgres3") self.postgres4_logs_dir = os.path.join(self.postgres_dir, "postgres4") + self.postgres_id = self.get_instance_docker_id(self.postgres_host) + + # available when with_postgresql_java_client = True + self.postgresql_java_client_host = "java" + self.postgresql_java_client_docker_id = self.get_instance_docker_id(self.postgresql_java_client_host) # available when with_mysql_client == True self.mysql_client_host = "mysql_client" @@ -573,7 +584,6 @@ class ClickHouseCluster: if p.exists(self.instances_dir): shutil.rmtree(self.instances_dir, ignore_errors=True) logging.debug(f"Removed :{self.instances_dir}") - os.mkdir(self.instances_dir) def print_all_docker_pieces(self): res_networks = subprocess.check_output( @@ -964,6 +974,26 @@ class ClickHouseCluster: p.join(docker_compose_yml_dir, "docker_compose_postgres_cluster.yml"), ] + def setup_postgresql_java_client_cmd( + self, instance, env_variables, docker_compose_yml_dir + ): + self.with_postgresql_java_client = True + self.base_cmd.extend( + [ + "--file", + p.join(docker_compose_yml_dir, "docker_compose_postgresql_java_client.yml"), + ] + ) + self.base_postgresql_java_client_cmd = [ + "docker-compose", + "--env-file", + instance.env_file, + "--project-name", + self.project_name, + "--file", + p.join(docker_compose_yml_dir, "docker_compose_postgresql_java_client.yml"), + ] + def setup_hdfs_cmd(self, instance, env_variables, docker_compose_yml_dir): self.with_hdfs = True env_variables["HDFS_HOST"] = self.hdfs_host @@ -1212,6 +1242,7 @@ class ClickHouseCluster: self.with_minio = True cert_d = p.join(self.minio_dir, "certs") env_variables["MINIO_CERTS_DIR"] = cert_d + env_variables["MINIO_DATA_DIR"] = self.minio_data_dir env_variables["MINIO_PORT"] = str(self.minio_port) env_variables["SSL_CERT_FILE"] = p.join(self.base_dir, cert_d, "public.crt") @@ -1335,6 +1366,7 @@ class ClickHouseCluster: with_odbc_drivers=False, with_postgres=False, with_postgres_cluster=False, + with_postgresql_java_client = False, with_hdfs=False, with_kerberized_hdfs=False, with_mongo=False, @@ -1360,6 +1392,7 @@ class ClickHouseCluster: tmpfs=None, zookeeper_docker_compose_path=None, minio_certs_dir=None, + minio_data_dir=None, use_keeper=True, main_config_name="config.xml", users_config_name="users.xml", @@ -1440,6 +1473,7 @@ class ClickHouseCluster: with_odbc_drivers=with_odbc_drivers, with_postgres=with_postgres, with_postgres_cluster=with_postgres_cluster, + with_postgresql_java_client=with_postgresql_java_client, hostname=hostname, env_variables=env_variables, image=image, @@ -1526,6 +1560,13 @@ class ClickHouseCluster: ) ) + if with_postgresql_java_client and not self.with_postgresql_java_client: + cmds.append( + self.setup_postgresql_java_client_cmd( + instance, env_variables, docker_compose_yml_dir + ) + ) + if with_odbc_drivers and not self.with_odbc_drivers: self.with_odbc_drivers = True if not self.with_mysql: @@ -1634,6 +1675,12 @@ class ClickHouseCluster: else: raise Exception("Overwriting minio certs dir") + if minio_data_dir is not None: + if self.minio_data_dir is None: + self.minio_data_dir = minio_data_dir + else: + raise Exception("Overwriting minio data dir") + if with_cassandra and not self.with_cassandra: cmds.append( self.setup_cassandra_cmd( @@ -2007,6 +2054,19 @@ class ClickHouseCluster: raise Exception("Cannot wait Postgres container") + def wait_postgresql_java_client(self, timeout=180): + start = time.time() + while time.time() - start < timeout: + try: + if check_postgresql_java_client_is_available(self.postgresql_java_client_docker_id): + logging.debug("PostgreSQL Java Client is available") + return True + time.sleep(0.5) + except Exception as ex: + logging.debug("Can't find PostgreSQL Java Client" + str(ex)) + time.sleep(0.5) + raise Exception("Cannot wait PostgreSQL Java Client container") + def wait_rabbitmq_to_start(self, timeout=180, throw=True): self.rabbitmq_ip = self.get_instance_ip(self.rabbitmq_host) @@ -2306,7 +2366,10 @@ class ClickHouseCluster: def start(self): pytest_xdist_logging_to_separate_files.setup() logging.info("Running tests in {}".format(self.base_path)) - + if not os.path.exists(self.instances_dir): + os.mkdir(self.instances_dir) + else: + logging.warning("Instance directory already exists. Did you call cluster.start() for second time?") logging.debug(f"Cluster start called. is_up={self.is_up}") self.print_all_docker_pieces() @@ -2441,7 +2504,7 @@ class ClickHouseCluster: self.wait_postgres_to_start() if self.with_postgres_cluster and self.base_postgres_cluster_cmd: - print("Setup Postgres") + logging.debug("Setup Postgres") os.makedirs(self.postgres2_logs_dir) os.chmod(self.postgres2_logs_dir, stat.S_IRWXU | stat.S_IRWXO) os.makedirs(self.postgres3_logs_dir) @@ -2452,6 +2515,12 @@ class ClickHouseCluster: self.up_called = True self.wait_postgres_cluster_to_start() + if self.with_postgresql_java_client and self.base_postgresql_java_client_cmd: + logging.debug("Setup Postgres Java Client") + subprocess_check_call(self.base_postgresql_java_client_cmd + common_opts) + self.up_called = True + self.wait_postgresql_java_client() + if self.with_kafka and self.base_kafka_cmd: logging.debug("Setup Kafka") subprocess_check_call( @@ -2575,6 +2644,8 @@ class ClickHouseCluster: os.path.join(self.base_dir, self.minio_certs_dir), os.path.join(self.minio_dir, "certs"), ) + os.mkdir(self.minio_data_dir) + os.chmod(self.minio_data_dir, stat.S_IRWXU | stat.S_IRWXO) minio_start_cmd = self.base_minio_cmd + common_opts @@ -2892,6 +2963,7 @@ class ClickHouseInstance: with_odbc_drivers, with_postgres, with_postgres_cluster, + with_postgresql_java_client, clickhouse_start_command=CLICKHOUSE_START_COMMAND, main_config_name="config.xml", users_config_name="users.xml", @@ -2953,6 +3025,7 @@ class ClickHouseInstance: self.with_mysql_cluster = with_mysql_cluster self.with_postgres = with_postgres self.with_postgres_cluster = with_postgres_cluster + self.with_postgresql_java_client = with_postgresql_java_client self.with_kafka = with_kafka self.with_kerberized_kafka = with_kerberized_kafka self.with_rabbitmq = with_rabbitmq diff --git a/tests/integration/test_postgresql_protocol/queries/query1.sql b/tests/integration/test_postgresql_protocol/queries/query1.sql new file mode 100644 index 00000000000..f9905d00ff1 --- /dev/null +++ b/tests/integration/test_postgresql_protocol/queries/query1.sql @@ -0,0 +1 @@ +SELECT 1 as a \ No newline at end of file diff --git a/tests/integration/test_postgresql_protocol/queries/query2.sql b/tests/integration/test_postgresql_protocol/queries/query2.sql new file mode 100644 index 00000000000..e1a24c29770 --- /dev/null +++ b/tests/integration/test_postgresql_protocol/queries/query2.sql @@ -0,0 +1 @@ +SELECT 'колонка' as a \ No newline at end of file diff --git a/tests/integration/test_postgresql_protocol/queries/query3.sql b/tests/integration/test_postgresql_protocol/queries/query3.sql new file mode 100644 index 00000000000..81f4a78b627 --- /dev/null +++ b/tests/integration/test_postgresql_protocol/queries/query3.sql @@ -0,0 +1 @@ +CREATE DATABASE x; USE x; CREATE TABLE table1 (column UInt32) ENGINE = Memory; INSERT INTO table1 VALUES (0), (1), (5); INSERT INTO table1 VALUES (0), (1), (5); SELECT * FROM table1 ORDER BY column; DROP DATABASE x; \ No newline at end of file diff --git a/tests/integration/test_postgresql_protocol/queries/query4.sql b/tests/integration/test_postgresql_protocol/queries/query4.sql new file mode 100644 index 00000000000..9d3ea218d80 --- /dev/null +++ b/tests/integration/test_postgresql_protocol/queries/query4.sql @@ -0,0 +1 @@ +CREATE TEMPORARY TABLE tmp (tmp_column UInt32); INSERT INTO tmp VALUES (0), (1); SELECT * FROM tmp ORDER BY tmp_column; DROP TABLE tmp; \ No newline at end of file diff --git a/tests/integration/test_postgresql_protocol/test.py b/tests/integration/test_postgresql_protocol/test.py index 43528c13c4d..c2bc1dc80f9 100644 --- a/tests/integration/test_postgresql_protocol/test.py +++ b/tests/integration/test_postgresql_protocol/test.py @@ -3,11 +3,9 @@ import datetime import decimal import os -import sys -import time import uuid +import logging -import docker import psycopg2 as py_psql import psycopg2.extras import pytest @@ -19,7 +17,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DOCKER_COMPOSE_PATH = get_docker_compose_path() cluster = ClickHouseCluster(__file__) -node = cluster.add_instance( +cluster.add_instance( "node", main_configs=[ "configs/postresql.xml", @@ -30,142 +28,69 @@ node = cluster.add_instance( "configs/server.key", ], user_configs=["configs/default_passwd.xml"], + with_postgres=True, + with_postgresql_java_client=True, env_variables={"UBSAN_OPTIONS": "print_stacktrace=1"}, ) server_port = 5433 - @pytest.fixture(scope="module") -def server_address(): - cluster.start() +def started_cluster(): try: - yield cluster.get_instance_ip("node") + cluster.start() + + yield cluster + except Exception as ex: + logging.exception(ex) + raise ex finally: cluster.shutdown() +def test_psql_client(started_cluster): + node = cluster.instances["node"] -@pytest.fixture(scope="module") -def psql_client(): - docker_compose = os.path.join(DOCKER_COMPOSE_PATH, "docker_compose_postgresql.yml") - run_and_check( - [ - "docker-compose", - "-p", - cluster.project_name, - "-f", - docker_compose, - "up", - "--force-recreate", - "-d", - "--build", - ] + for query_file in ['query1.sql', 'query2.sql', 'query3.sql', 'query4.sql']: + started_cluster.copy_file_to_container(started_cluster.postgres_id, os.path.join(SCRIPT_DIR, 'queries', query_file), f'/{query_file}') + cmd_prefix = ['/usr/bin/psql', f"sslmode=require host={node.hostname} port={server_port} user=default dbname=default password=123"] + cmd_prefix += ["--no-align", "--field-separator=' '"] + + res = started_cluster.exec_in_container(started_cluster.postgres_id, + cmd_prefix + ['-f', '/query1.sql'], + shell=True ) - yield docker.DockerClient( - base_url="unix:///var/run/docker.sock", - version=cluster.docker_api_version, - timeout=600, - ).containers.get(cluster.project_name + "_psql_1") + logging.debug(res) + assert res == "\n".join(["a", "1", "(1 row)", ""]) - -@pytest.fixture(scope="module") -def psql_server(psql_client): - """Return PostgreSQL container when it is healthy.""" - retries = 30 - for i in range(retries): - info = psql_client.client.api.inspect_container(psql_client.name) - if info["State"]["Health"]["Status"] == "healthy": - break - time.sleep(1) - else: - print(info["State"]) - raise Exception( - "PostgreSQL server has not started after {} retries.".format(retries) - ) - - return psql_client - - -@pytest.fixture(scope="module") -def java_container(): - docker_compose = os.path.join( - DOCKER_COMPOSE_PATH, "docker_compose_postgresql_java_client.yml" + res = started_cluster.exec_in_container(started_cluster.postgres_id, + cmd_prefix + ['-f', '/query2.sql'], + shell=True ) - run_and_check( - [ - "docker-compose", - "-p", - cluster.project_name, - "-f", - docker_compose, - "up", - "--force-recreate", - "-d", - "--build", - ] + logging.debug(res) + assert res == "\n".join(["a", "колонка", "(1 row)", ""]) + + res = started_cluster.exec_in_container(started_cluster.postgres_id, + cmd_prefix + ['-f', '/query3.sql'], + shell=True ) - yield docker.DockerClient( - base_url="unix:///var/run/docker.sock", - version=cluster.docker_api_version, - timeout=600, - ).containers.get(cluster.project_name + "_java_1") - - -def test_psql_is_ready(psql_server): - pass - - -def test_psql_client(psql_client, server_address): - cmd_prefix = 'psql "sslmode=require host={server_address} port={server_port} user=default dbname=default password=123" '.format( - server_address=server_address, server_port=server_port - ) - cmd_prefix += "--no-align --field-separator=' ' " - - code, (stdout, stderr) = psql_client.exec_run( - cmd_prefix + '-c "SELECT 1 as a"', demux=True - ) - assert stdout.decode() == "\n".join(["a", "1", "(1 row)", ""]) - - code, (stdout, stderr) = psql_client.exec_run( - cmd_prefix + '''-c "SELECT 'колонка' as a"''', demux=True - ) - assert stdout.decode() == "\n".join(["a", "колонка", "(1 row)", ""]) - - code, (stdout, stderr) = psql_client.exec_run( - cmd_prefix - + "-c " - + """ - "CREATE DATABASE x; - USE x; - CREATE TABLE table1 (column UInt32) ENGINE = Memory; - INSERT INTO table1 VALUES (0), (1), (5); - INSERT INTO table1 VALUES (0), (1), (5); - SELECT * FROM table1 ORDER BY column;" - """, - demux=True, - ) - assert stdout.decode() == "\n".join( - ["column", "0", "0", "1", "1", "5", "5", "(6 rows)", ""] + logging.debug(res) + assert res == "\n".join( + ["SELECT 0","SELECT 0","SELECT 0","INSERT 0 0", "INSERT 0 0", "column", "0", "0", "1", "1", "5", "5", "(6 rows)", "SELECT 0\n"] ) - code, (stdout, stderr) = psql_client.exec_run( - cmd_prefix - + "-c " - + """ - "DROP DATABASE x; - CREATE TEMPORARY TABLE tmp (tmp_column UInt32); - INSERT INTO tmp VALUES (0), (1); - SELECT * FROM tmp ORDER BY tmp_column;" - """, - demux=True, + res = started_cluster.exec_in_container(started_cluster.postgres_id, + cmd_prefix + ['-f', '/query4.sql'], + shell=True ) - assert stdout.decode() == "\n".join(["tmp_column", "0", "1", "(2 rows)", ""]) + logging.debug(res) + assert res == "\n".join(["SELECT 0","INSERT 0 0","tmp_column", "0", "1", "(2 rows)", "SELECT 0\n"]) +def test_python_client(started_cluster): + node = cluster.instances["node"] -def test_python_client(server_address): with pytest.raises(py_psql.InternalError) as exc_info: ch = py_psql.connect( - host=server_address, + host=node.ip_address, port=server_port, user="default", password="123", @@ -179,7 +104,7 @@ def test_python_client(server_address): ) ch = py_psql.connect( - host=server_address, + host=node.ip_address, port=server_port, user="default", password="123", @@ -209,26 +134,26 @@ def test_python_client(server_address): decimal.Decimal("0.3333330000"), uuid.UUID("61f0c404-5cb3-11e7-907b-a6006ad3dba0"), ) + cur.execute( + "DROP DATABASE x" + ) -def test_java_client(server_address, java_container): +def test_java_client(started_cluster): + node = cluster.instances["node"] + with open(os.path.join(SCRIPT_DIR, "java.reference")) as fp: reference = fp.read() # database not exists exception. - code, (stdout, stderr) = java_container.exec_run( - "java JavaConnectorTest --host {host} --port {port} --user default --database " - "abc".format(host=server_address, port=server_port), - demux=True, - ) - assert code == 1 + with pytest.raises(Exception) as exc: + res = started_cluster.exec_in_container(started_cluster.postgresql_java_client_docker_id, + ["bash", "-c", f"java JavaConnectorTest --host {node.hostname} --port {server_port} --user default --database abc"], + ) + assert 'org.postgresql.util.PSQLException: ERROR: Invalid user or password' in str(exc.value) # non-empty password passed. - code, (stdout, stderr) = java_container.exec_run( - "java JavaConnectorTest --host {host} --port {port} --user default --password 123 --database " - "default".format(host=server_address, port=server_port), - demux=True, + res= started_cluster.exec_in_container(started_cluster.postgresql_java_client_docker_id, + ["bash", "-c", f"java JavaConnectorTest --host {node.hostname} --port {server_port} --user default --password 123 --database default"] ) - print(stdout, stderr, file=sys.stderr) - assert code == 0 - assert stdout.decode() == reference + assert res == reference From bbc02c0762ea11eee5ffd2cf1fa51d5f90e9e8b2 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Jan 2023 21:28:54 +0000 Subject: [PATCH 43/88] black --- tests/integration/helpers/cluster.py | 29 +++++-- .../test_postgresql_protocol/test.py | 86 +++++++++++++------ 2 files changed, 82 insertions(+), 33 deletions(-) diff --git a/tests/integration/helpers/cluster.py b/tests/integration/helpers/cluster.py index eabbc489ba2..5a9f8b20254 100644 --- a/tests/integration/helpers/cluster.py +++ b/tests/integration/helpers/cluster.py @@ -206,6 +206,7 @@ def check_kafka_is_available(kafka_id, kafka_port): p.communicate() return p.returncode == 0 + def check_postgresql_java_client_is_available(postgresql_java_client_id): p = subprocess.Popen( ("docker", "exec", "-i", postgresql_java_client_id, "java", "-version"), @@ -214,6 +215,7 @@ def check_postgresql_java_client_is_available(postgresql_java_client_id): p.communicate() return p.returncode == 0 + def check_rabbitmq_is_available(rabbitmq_id): p = subprocess.Popen( ("docker", "exec", "-i", rabbitmq_id, "rabbitmqctl", "await_startup"), @@ -526,7 +528,9 @@ class ClickHouseCluster: # available when with_postgresql_java_client = True self.postgresql_java_client_host = "java" - self.postgresql_java_client_docker_id = self.get_instance_docker_id(self.postgresql_java_client_host) + self.postgresql_java_client_docker_id = self.get_instance_docker_id( + self.postgresql_java_client_host + ) # available when with_mysql_client == True self.mysql_client_host = "mysql_client" @@ -981,7 +985,9 @@ class ClickHouseCluster: self.base_cmd.extend( [ "--file", - p.join(docker_compose_yml_dir, "docker_compose_postgresql_java_client.yml"), + p.join( + docker_compose_yml_dir, "docker_compose_postgresql_java_client.yml" + ), ] ) self.base_postgresql_java_client_cmd = [ @@ -1366,7 +1372,7 @@ class ClickHouseCluster: with_odbc_drivers=False, with_postgres=False, with_postgres_cluster=False, - with_postgresql_java_client = False, + with_postgresql_java_client=False, with_hdfs=False, with_kerberized_hdfs=False, with_mongo=False, @@ -2058,7 +2064,9 @@ class ClickHouseCluster: start = time.time() while time.time() - start < timeout: try: - if check_postgresql_java_client_is_available(self.postgresql_java_client_docker_id): + if check_postgresql_java_client_is_available( + self.postgresql_java_client_docker_id + ): logging.debug("PostgreSQL Java Client is available") return True time.sleep(0.5) @@ -2369,7 +2377,9 @@ class ClickHouseCluster: if not os.path.exists(self.instances_dir): os.mkdir(self.instances_dir) else: - logging.warning("Instance directory already exists. Did you call cluster.start() for second time?") + logging.warning( + "Instance directory already exists. Did you call cluster.start() for second time?" + ) logging.debug(f"Cluster start called. is_up={self.is_up}") self.print_all_docker_pieces() @@ -2515,9 +2525,14 @@ class ClickHouseCluster: self.up_called = True self.wait_postgres_cluster_to_start() - if self.with_postgresql_java_client and self.base_postgresql_java_client_cmd: + if ( + self.with_postgresql_java_client + and self.base_postgresql_java_client_cmd + ): logging.debug("Setup Postgres Java Client") - subprocess_check_call(self.base_postgresql_java_client_cmd + common_opts) + subprocess_check_call( + self.base_postgresql_java_client_cmd + common_opts + ) self.up_called = True self.wait_postgresql_java_client() diff --git a/tests/integration/test_postgresql_protocol/test.py b/tests/integration/test_postgresql_protocol/test.py index c2bc1dc80f9..e1d8cbf9bcc 100644 --- a/tests/integration/test_postgresql_protocol/test.py +++ b/tests/integration/test_postgresql_protocol/test.py @@ -35,6 +35,7 @@ cluster.add_instance( server_port = 5433 + @pytest.fixture(scope="module") def started_cluster(): try: @@ -47,43 +48,65 @@ def started_cluster(): finally: cluster.shutdown() + def test_psql_client(started_cluster): node = cluster.instances["node"] - for query_file in ['query1.sql', 'query2.sql', 'query3.sql', 'query4.sql']: - started_cluster.copy_file_to_container(started_cluster.postgres_id, os.path.join(SCRIPT_DIR, 'queries', query_file), f'/{query_file}') - cmd_prefix = ['/usr/bin/psql', f"sslmode=require host={node.hostname} port={server_port} user=default dbname=default password=123"] + for query_file in ["query1.sql", "query2.sql", "query3.sql", "query4.sql"]: + started_cluster.copy_file_to_container( + started_cluster.postgres_id, + os.path.join(SCRIPT_DIR, "queries", query_file), + f"/{query_file}", + ) + cmd_prefix = [ + "/usr/bin/psql", + f"sslmode=require host={node.hostname} port={server_port} user=default dbname=default password=123", + ] cmd_prefix += ["--no-align", "--field-separator=' '"] - res = started_cluster.exec_in_container(started_cluster.postgres_id, - cmd_prefix + ['-f', '/query1.sql'], - shell=True + res = started_cluster.exec_in_container( + started_cluster.postgres_id, cmd_prefix + ["-f", "/query1.sql"], shell=True ) logging.debug(res) assert res == "\n".join(["a", "1", "(1 row)", ""]) - res = started_cluster.exec_in_container(started_cluster.postgres_id, - cmd_prefix + ['-f', '/query2.sql'], - shell=True + res = started_cluster.exec_in_container( + started_cluster.postgres_id, cmd_prefix + ["-f", "/query2.sql"], shell=True ) logging.debug(res) assert res == "\n".join(["a", "колонка", "(1 row)", ""]) - res = started_cluster.exec_in_container(started_cluster.postgres_id, - cmd_prefix + ['-f', '/query3.sql'], - shell=True + res = started_cluster.exec_in_container( + started_cluster.postgres_id, cmd_prefix + ["-f", "/query3.sql"], shell=True ) logging.debug(res) assert res == "\n".join( - ["SELECT 0","SELECT 0","SELECT 0","INSERT 0 0", "INSERT 0 0", "column", "0", "0", "1", "1", "5", "5", "(6 rows)", "SELECT 0\n"] + [ + "SELECT 0", + "SELECT 0", + "SELECT 0", + "INSERT 0 0", + "INSERT 0 0", + "column", + "0", + "0", + "1", + "1", + "5", + "5", + "(6 rows)", + "SELECT 0\n", + ] ) - res = started_cluster.exec_in_container(started_cluster.postgres_id, - cmd_prefix + ['-f', '/query4.sql'], - shell=True + res = started_cluster.exec_in_container( + started_cluster.postgres_id, cmd_prefix + ["-f", "/query4.sql"], shell=True ) logging.debug(res) - assert res == "\n".join(["SELECT 0","INSERT 0 0","tmp_column", "0", "1", "(2 rows)", "SELECT 0\n"]) + assert res == "\n".join( + ["SELECT 0", "INSERT 0 0", "tmp_column", "0", "1", "(2 rows)", "SELECT 0\n"] + ) + def test_python_client(started_cluster): node = cluster.instances["node"] @@ -134,9 +157,7 @@ def test_python_client(started_cluster): decimal.Decimal("0.3333330000"), uuid.UUID("61f0c404-5cb3-11e7-907b-a6006ad3dba0"), ) - cur.execute( - "DROP DATABASE x" - ) + cur.execute("DROP DATABASE x") def test_java_client(started_cluster): @@ -147,13 +168,26 @@ def test_java_client(started_cluster): # database not exists exception. with pytest.raises(Exception) as exc: - res = started_cluster.exec_in_container(started_cluster.postgresql_java_client_docker_id, - ["bash", "-c", f"java JavaConnectorTest --host {node.hostname} --port {server_port} --user default --database abc"], - ) - assert 'org.postgresql.util.PSQLException: ERROR: Invalid user or password' in str(exc.value) + res = started_cluster.exec_in_container( + started_cluster.postgresql_java_client_docker_id, + [ + "bash", + "-c", + f"java JavaConnectorTest --host {node.hostname} --port {server_port} --user default --database abc", + ], + ) + assert ( + "org.postgresql.util.PSQLException: ERROR: Invalid user or password" + in str(exc.value) + ) # non-empty password passed. - res= started_cluster.exec_in_container(started_cluster.postgresql_java_client_docker_id, - ["bash", "-c", f"java JavaConnectorTest --host {node.hostname} --port {server_port} --user default --password 123 --database default"] + res = started_cluster.exec_in_container( + started_cluster.postgresql_java_client_docker_id, + [ + "bash", + "-c", + f"java JavaConnectorTest --host {node.hostname} --port {server_port} --user default --password 123 --database default", + ], ) assert res == reference From bb288c1f14fc383a6a0cfd9d3a27b7c983ab2759 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Tue, 3 Jan 2023 21:38:59 +0000 Subject: [PATCH 44/88] how many more? --- tests/integration/ci-runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index b94e6a56e2c..9d9d60d43e6 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -316,7 +316,7 @@ class ClickhouseIntegrationTestsRunner: except subprocess.CalledProcessError as err: logging.info("docker-compose pull failed: " + str(err)) return - logging.error("Pulling %s failed for 5 attempts. Will fail the worker.", image) + logging.error("Pulling images failed for 5 attempts. Will fail the worker.") exit(1) def _can_run_with(self, path, opt): From 4f0b0236e32a4312724161020a7be8bf28495222 Mon Sep 17 00:00:00 2001 From: Smita Kulkarni Date: Wed, 4 Jan 2023 09:56:44 +0100 Subject: [PATCH 45/88] Removed flag from ExpressionAnalyzer and added a check in MutationsInterpreter::addStreamsForLaterStages - 42637 Disallow arrayJoin in mutations --- src/Interpreters/ActionsVisitor.cpp | 7 +------ src/Interpreters/ActionsVisitor.h | 4 +--- src/Interpreters/ExpressionAnalyzer.cpp | 10 ++++------ src/Interpreters/ExpressionAnalyzer.h | 4 ++-- src/Interpreters/MutationsInterpreter.cpp | 8 +++++--- 5 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/Interpreters/ActionsVisitor.cpp b/src/Interpreters/ActionsVisitor.cpp index 5dfb252fa18..9a0d33b19fc 100644 --- a/src/Interpreters/ActionsVisitor.cpp +++ b/src/Interpreters/ActionsVisitor.cpp @@ -535,8 +535,7 @@ ActionsMatcher::Data::Data( bool only_consts_, bool create_source_for_in_, AggregationKeysInfo aggregation_keys_info_, - bool build_expression_with_window_functions_, - bool disallow_arrayjoin_) + bool build_expression_with_window_functions_) : WithContext(context_) , set_size_limit(set_size_limit_) , subquery_depth(subquery_depth_) @@ -550,7 +549,6 @@ ActionsMatcher::Data::Data( , actions_stack(std::move(actions_dag), context_) , aggregation_keys_info(aggregation_keys_info_) , build_expression_with_window_functions(build_expression_with_window_functions_) - , disallow_arrayjoin(disallow_arrayjoin_) , next_unique_suffix(actions_stack.getLastActions().getOutputs().size() + 1) { } @@ -889,9 +887,6 @@ void ActionsMatcher::visit(const ASTFunction & node, const ASTPtr & ast, Data & /// Function arrayJoin. if (node.name == "arrayJoin") { - if (data.disallow_arrayjoin) - throw Exception("arrayJoin is disallowed in mutations", ErrorCodes::UNEXPECTED_EXPRESSION); - if (node.arguments->children.size() != 1) throw Exception("arrayJoin requires exactly 1 argument", ErrorCodes::TYPE_MISMATCH); diff --git a/src/Interpreters/ActionsVisitor.h b/src/Interpreters/ActionsVisitor.h index 16b32162bfe..fea013fd075 100644 --- a/src/Interpreters/ActionsVisitor.h +++ b/src/Interpreters/ActionsVisitor.h @@ -134,7 +134,6 @@ public: ScopeStack actions_stack; AggregationKeysInfo aggregation_keys_info; bool build_expression_with_window_functions; - bool disallow_arrayjoin; /* * Remember the last unique column suffix to avoid quadratic behavior @@ -155,8 +154,7 @@ public: bool only_consts_, bool create_source_for_in_, AggregationKeysInfo aggregation_keys_info_, - bool build_expression_with_window_functions_ = false, - bool disallow_arrayjoin = false); + bool build_expression_with_window_functions_ = false); /// Does result of the calculation already exists in the block. bool hasColumn(const String & column_name) const; diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index 7478c749d7d..22229c0d6c2 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -538,7 +538,7 @@ void SelectQueryExpressionAnalyzer::makeSetsForIndex(const ASTPtr & node) } -void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts, bool disallow_arrayJoin) +void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts) { LogAST log; ActionsVisitor::Data visitor_data( @@ -552,9 +552,7 @@ void ExpressionAnalyzer::getRootActions(const ASTPtr & ast, bool no_makeset_for_ false /* no_makeset */, only_consts, !isRemoteStorage() /* create_source_for_in */, - getAggregationKeysInfo(), - false /*build_expression_with_window_functions */, - disallow_arrayJoin); + getAggregationKeysInfo()); ActionsVisitor(visitor_data, log.stream()).visit(ast); actions = visitor_data.getActions(); } @@ -1713,10 +1711,10 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio } -void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types, bool disallow_arrayJoin) +void ExpressionAnalyzer::appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types) { ExpressionActionsChain::Step & step = chain.lastStep(sourceColumns()); - getRootActions(expr, only_types, step.actions(), false /* only_consts */, disallow_arrayJoin); + getRootActions(expr, only_types, step.actions()); step.addRequiredOutput(expr->getColumnName()); } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index cb376ab6326..ddb41a00f84 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -110,7 +110,7 @@ public: ~ExpressionAnalyzer(); - void appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types, bool disallow_arrayJoin = false); + void appendExpression(ExpressionActionsChain & chain, const ASTPtr & expr, bool only_types); /// If `ast` is not a SELECT query, just gets all the actions to evaluate the expression. /// If add_aliases, only the calculated values in the desired order and add aliases. @@ -175,7 +175,7 @@ protected: ArrayJoinActionPtr addMultipleArrayJoinAction(ActionsDAGPtr & actions, bool is_left) const; - void getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts = false, bool disallow_arrayJoin = false); + void getRootActions(const ASTPtr & ast, bool no_makeset_for_subqueries, ActionsDAGPtr & actions, bool only_consts = false); /** Similar to getRootActions but do not make sets when analyzing IN functions. It's used in * analyzeAggregation which happens earlier than analyzing PREWHERE and WHERE. If we did, the diff --git a/src/Interpreters/MutationsInterpreter.cpp b/src/Interpreters/MutationsInterpreter.cpp index 89e85b6a7da..0552c41fce7 100644 --- a/src/Interpreters/MutationsInterpreter.cpp +++ b/src/Interpreters/MutationsInterpreter.cpp @@ -43,6 +43,7 @@ namespace ErrorCodes extern const int UNKNOWN_MUTATION_COMMAND; extern const int NO_SUCH_COLUMN_IN_TABLE; extern const int CANNOT_UPDATE_COLUMN; + extern const int UNEXPECTED_EXPRESSION; } namespace @@ -861,7 +862,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & { if (!actions_chain.steps.empty()) actions_chain.addStep(); - stage.analyzer->appendExpression(actions_chain, ast, dry_run, true /* disallow_arrayJoin */); + stage.analyzer->appendExpression(actions_chain, ast, dry_run); stage.filter_column_names.push_back(ast->getColumnName()); } @@ -871,7 +872,7 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & actions_chain.addStep(); for (const auto & kv : stage.column_to_updated) - stage.analyzer->appendExpression(actions_chain, kv.second, dry_run, true /* disallow_arrayJoin */); + stage.analyzer->appendExpression(actions_chain, kv.second, dry_run); auto & actions = actions_chain.getLastStep().actions(); @@ -884,7 +885,6 @@ ASTPtr MutationsInterpreter::prepareInterpreterSelectQuery(std::vector & } } - /// Remove all intermediate columns. actions_chain.addStep(); actions_chain.getLastStep().required_output.clear(); @@ -942,6 +942,8 @@ QueryPipelineBuilder MutationsInterpreter::addStreamsForLaterStages(const std::v for (size_t i = 0; i < stage.expressions_chain.steps.size(); ++i) { const auto & step = stage.expressions_chain.steps[i]; + if (step->actions()->hasArrayJoin()) + throw Exception("arrayJoin is not allowed in mutations", ErrorCodes::UNEXPECTED_EXPRESSION); if (i < stage.filter_column_names.size()) { /// Execute DELETEs. From 05ce5eec06ee8111665ccd9d954cf00774bfb472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Wed, 4 Jan 2023 12:01:39 +0100 Subject: [PATCH 46/88] Ignore hwmon sensors on label read issues --- src/Common/AsynchronousMetrics.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Common/AsynchronousMetrics.cpp b/src/Common/AsynchronousMetrics.cpp index b68fcab2449..1ef00c157c0 100644 --- a/src/Common/AsynchronousMetrics.cpp +++ b/src/Common/AsynchronousMetrics.cpp @@ -213,18 +213,18 @@ void AsynchronousMetrics::openSensorsChips() if (!file) continue; - String sensor_name; - if (sensor_name_file_exists) - { - ReadBufferFromFilePRead sensor_name_in(sensor_name_file, small_buffer_size); - readText(sensor_name, sensor_name_in); - std::replace(sensor_name.begin(), sensor_name.end(), ' ', '_'); - } - - file->rewind(); - Int64 temperature = 0; + String sensor_name{}; try { + if (sensor_name_file_exists) + { + ReadBufferFromFilePRead sensor_name_in(sensor_name_file, small_buffer_size); + readText(sensor_name, sensor_name_in); + std::replace(sensor_name.begin(), sensor_name.end(), ' ', '_'); + } + + file->rewind(); + Int64 temperature = 0; readText(temperature, *file); } catch (const ErrnoException & e) @@ -233,7 +233,7 @@ void AsynchronousMetrics::openSensorsChips() &Poco::Logger::get("AsynchronousMetrics"), "Hardware monitor '{}', sensor '{}' exists but could not be read: {}.", hwmon_name, - sensor_name, + sensor_index, errnoToString(e.getErrno())); continue; } From 169580da6933551e49f5bda85deb651eb8acbcf2 Mon Sep 17 00:00:00 2001 From: Enrique Herreros Date: Tue, 3 Jan 2023 21:38:15 -0500 Subject: [PATCH 47/88] typo --- src/Interpreters/replaceForPositionalArguments.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Interpreters/replaceForPositionalArguments.cpp b/src/Interpreters/replaceForPositionalArguments.cpp index 410cc5bb10c..241dd7cf92c 100644 --- a/src/Interpreters/replaceForPositionalArguments.cpp +++ b/src/Interpreters/replaceForPositionalArguments.cpp @@ -33,7 +33,7 @@ bool replaceForPositionalArguments(ASTPtr & argument, const ASTSelectQuery * sel auto pos = ast_literal->value.get(); if (!pos || pos > columns.size()) throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, - "Positional argument out of bounds: {} (exprected in range [1, {}]", + "Positional argument out of bounds: {} (expected in range [1, {}]", pos, columns.size()); const auto & column = columns[--pos]; From 08fbf6e5b23f3d72d2c4f5d5c5cb8edbc3e71c3a Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Jan 2023 11:48:28 +0000 Subject: [PATCH 48/88] again --- tests/integration/ci-runner.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 9d9d60d43e6..4913abd8dad 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -307,7 +307,7 @@ class ClickhouseIntegrationTestsRunner: ) for i in range(5): - logging.info("Pulling images before running tests. Attempt %s", image, i) + logging.info("Pulling images before running tests. Attempt %s", i) try: subprocess.check_output( cmd, From cebfddc3ad2c9ce69d94a9937a1ba641a3715558 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Jan 2023 12:54:11 +0100 Subject: [PATCH 49/88] Better closing of the loop --- src/Storages/RabbitMQ/RabbitMQConnection.cpp | 19 +++++++++++ src/Storages/RabbitMQ/RabbitMQConnection.h | 2 ++ src/Storages/UVLoop.h | 35 +++++++++++++++++++- 3 files changed, 55 insertions(+), 1 deletion(-) diff --git a/src/Storages/RabbitMQ/RabbitMQConnection.cpp b/src/Storages/RabbitMQ/RabbitMQConnection.cpp index 13d065774a2..a290d796e32 100644 --- a/src/Storages/RabbitMQ/RabbitMQConnection.cpp +++ b/src/Storages/RabbitMQ/RabbitMQConnection.cpp @@ -122,4 +122,23 @@ void RabbitMQConnection::disconnectImpl(bool immediately) event_handler.iterateLoop(); } +RabbitMQConnection::~RabbitMQConnection() +{ + std::lock_guard lock(mutex); + + if (!connection) + return; + + try + { + /// Try to always close the connection gracefully (run the loop to see the closing callbacks) + /// to make sure that the associated callbacks and pending events are removed. + disconnectImpl(); + } + catch (...) + { + tryLogCurrentException(__PRETTY_FUNCTION__); + } +} + } diff --git a/src/Storages/RabbitMQ/RabbitMQConnection.h b/src/Storages/RabbitMQ/RabbitMQConnection.h index 7a355afea0e..288b8ccc465 100644 --- a/src/Storages/RabbitMQ/RabbitMQConnection.h +++ b/src/Storages/RabbitMQ/RabbitMQConnection.h @@ -24,6 +24,8 @@ class RabbitMQConnection public: RabbitMQConnection(const RabbitMQConfiguration & configuration_, Poco::Logger * log_); + ~RabbitMQConnection(); + bool isConnected(); bool connect(); diff --git a/src/Storages/UVLoop.h b/src/Storages/UVLoop.h index 66668739dd7..3a0e2df343e 100644 --- a/src/Storages/UVLoop.h +++ b/src/Storages/UVLoop.h @@ -6,6 +6,7 @@ #include #include +#include namespace DB { @@ -30,7 +31,38 @@ public: ~UVLoop() { if (loop_ptr) - uv_loop_close(loop_ptr.get()); + { + auto res = uv_loop_close(loop_ptr.get()); + + if (res == UV_EBUSY) + { + LOG_DEBUG(log, "Closing pending handles"); + uv_walk(loop_ptr.get(), [](uv_handle_t * handle, void *) + { + if (!uv_is_closing(handle)) + uv_close(handle, [](uv_handle_t *){}); + }, nullptr); + + /// Run the loop until there are no pending callbacks. + while (true) + { + res = uv_run(loop_ptr.get(), UV_RUN_ONCE); + if (res) + LOG_DEBUG(log, "Waiting for pending callbacks to finish ({})", res); + else + break; + } + + res = uv_loop_close(loop_ptr.get()); + if (res == UV_EBUSY) + { + LOG_ERROR( + log, "Failed to close libuv loop (active requests/handles in the loop: {})", + uv_loop_alive(loop_ptr.get())); + chassert(false); + } + } + } } inline uv_loop_t * getLoop() { return loop_ptr.get(); } @@ -39,6 +71,7 @@ public: private: std::unique_ptr loop_ptr; + Poco::Logger * log = &Poco::Logger::get("UVLoop"); }; } From ae7a5a65545908aad8e26bd9ada3a9dcf91b1714 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Jan 2023 13:07:56 +0100 Subject: [PATCH 50/88] Better comment --- src/Storages/RabbitMQ/RabbitMQConnection.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Storages/RabbitMQ/RabbitMQConnection.cpp b/src/Storages/RabbitMQ/RabbitMQConnection.cpp index a290d796e32..39aa2b87caf 100644 --- a/src/Storages/RabbitMQ/RabbitMQConnection.cpp +++ b/src/Storages/RabbitMQ/RabbitMQConnection.cpp @@ -132,7 +132,8 @@ RabbitMQConnection::~RabbitMQConnection() try { /// Try to always close the connection gracefully (run the loop to see the closing callbacks) - /// to make sure that the associated callbacks and pending events are removed. + /// to make sure that the associated callbacks and pending events are removed + /// before handler and loop are destructed. disconnectImpl(); } catch (...) From 0fe4d0732d43f273f2119099049155c6f2967400 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Jan 2023 13:17:40 +0100 Subject: [PATCH 51/88] Better --- src/Storages/UVLoop.h | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/Storages/UVLoop.h b/src/Storages/UVLoop.h index 3a0e2df343e..f5f8d715170 100644 --- a/src/Storages/UVLoop.h +++ b/src/Storages/UVLoop.h @@ -37,11 +37,7 @@ public: if (res == UV_EBUSY) { LOG_DEBUG(log, "Closing pending handles"); - uv_walk(loop_ptr.get(), [](uv_handle_t * handle, void *) - { - if (!uv_is_closing(handle)) - uv_close(handle, [](uv_handle_t *){}); - }, nullptr); + uv_walk(loop_ptr.get(), onUVWalkClosingCallback, nullptr); /// Run the loop until there are no pending callbacks. while (true) @@ -72,6 +68,14 @@ public: private: std::unique_ptr loop_ptr; Poco::Logger * log = &Poco::Logger::get("UVLoop"); + + static void onUVWalkClosingCallback(uv_handle_t * handle, void *) + { + if (!uv_is_closing(handle)) + uv_close(handle, onUVCloseCallback); + } + + static void onUVCloseCallback(uv_handle_t *) {} }; } From 283388cfa6c155120c865094da62b427ce484077 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Jan 2023 13:43:33 +0100 Subject: [PATCH 52/88] Simplify loop, add comment --- src/Storages/RabbitMQ/RabbitMQConnection.h | 4 +++- src/Storages/UVLoop.h | 8 ++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQConnection.h b/src/Storages/RabbitMQ/RabbitMQConnection.h index 288b8ccc465..ac9380d25db 100644 --- a/src/Storages/RabbitMQ/RabbitMQConnection.h +++ b/src/Storages/RabbitMQ/RabbitMQConnection.h @@ -56,9 +56,11 @@ private: Poco::Logger * log; UVLoop loop; + /// Preserve order of destruction here: + /// destruct connection and handler before the loop above. RabbitMQHandler event_handler; - std::unique_ptr connection; + std::mutex mutex; }; diff --git a/src/Storages/UVLoop.h b/src/Storages/UVLoop.h index f5f8d715170..6b24252077e 100644 --- a/src/Storages/UVLoop.h +++ b/src/Storages/UVLoop.h @@ -40,13 +40,9 @@ public: uv_walk(loop_ptr.get(), onUVWalkClosingCallback, nullptr); /// Run the loop until there are no pending callbacks. - while (true) + while ((res = uv_run(loop_ptr.get(), UV_RUN_ONCE)) != 0) { - res = uv_run(loop_ptr.get(), UV_RUN_ONCE); - if (res) - LOG_DEBUG(log, "Waiting for pending callbacks to finish ({})", res); - else - break; + LOG_DEBUG(log, "Waiting for pending callbacks to finish ({})", res); } res = uv_loop_close(loop_ptr.get()); From 4d5131ed7dba83a6af75addf7bc8adfecbc64739 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Jan 2023 13:16:31 +0000 Subject: [PATCH 53/88] debug quotes --- tests/integration/ci-runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 4913abd8dad..de3bd680002 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -298,11 +298,11 @@ class ClickhouseIntegrationTestsRunner: cmd = ( "cd {repo_path}/tests/integration && " - "timeout -s 9 1h ./runner {runner_opts} {image_cmd} {command} ".format( + "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --command '{command}' ".format( repo_path=repo_path, runner_opts=self._get_runner_opts(), image_cmd=image_cmd, - command=r""" find /compose -name 'docker_compose_*.yml' -exec docker-compose -f '{}' pull \; """, + command=r""" find /compose -name docker_compose_*.yml -exec docker-compose -f {} pull \; """, ) ) @@ -313,9 +313,10 @@ class ClickhouseIntegrationTestsRunner: cmd, shell=True, ) + return except subprocess.CalledProcessError as err: logging.info("docker-compose pull failed: " + str(err)) - return + continue logging.error("Pulling images failed for 5 attempts. Will fail the worker.") exit(1) From 1a1afc6a74d773574dca68eaf5d99ed11900d4e2 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 4 Jan 2023 14:18:35 +0000 Subject: [PATCH 54/88] Make test_concurrent_queries_restriction_by_query_kind less flaky --- .../test.py | 39 ++++++++++++++----- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py b/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py index 6bda1df147c..97f39c60004 100644 --- a/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py +++ b/tests/integration/test_concurrent_queries_restriction_by_query_kind/test.py @@ -38,24 +38,43 @@ def execute_with_background(node, sql, background_sql, background_times, wait_ti time.sleep(1) else: assert False, "there are unknown background queries: {}".format(r) + + query_results = [] for _ in range(background_times): - node.get_query_request(background_sql, stdin="") - time.sleep(0.5) # wait background to start. - return node.query(sql, stdin="") + query_results.append(node.get_query_request(background_sql, stdin="")) + + query_results.append(node.get_query_request(sql, stdin="")) + return query_results def common_pattern(node, query_kind, restricted_sql, normal_sql, limit, wait_times): - # restriction is working - with pytest.raises( - Exception, match=r".*Too many simultaneous {} queries.*".format(query_kind) - ): - execute_with_background(node, restricted_sql, restricted_sql, limit, wait_times) + query_results = execute_with_background( + node, restricted_sql, restricted_sql, limit, wait_times + ) + + errors = [query_result.get_answer_and_error()[1] for query_result in query_results] + assert ( + sum(1 for e in errors if f"Too many simultaneous {query_kind} queries" in e) + == 1 + ), f"Expected exactly 1 query to fail because of too many simultaneous queries, got errors: {errors}" + + def assert_all_queries_passed(query_resuts): + errors = [ + query_result.get_answer_and_error()[1] for query_result in query_results + ] + assert all( + len(e) == 0 for e in errors + ), f"Expected for all queries to pass, got errors: {errors}" # different query kind is independent - execute_with_background(node, normal_sql, restricted_sql, limit, wait_times) + query_results = execute_with_background( + node, normal_sql, restricted_sql, limit, wait_times + ) + assert_all_queries_passed(query_results) # normal - execute_with_background(node, restricted_sql, "", 0, wait_times) + query_results = execute_with_background(node, restricted_sql, "", 0, wait_times) + assert_all_queries_passed(query_results) def test_select(started_cluster): From 7c2cdfa65a6d8c2e7bcda213946586b3d84b37e7 Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Wed, 4 Jan 2023 15:46:38 +0000 Subject: [PATCH 55/88] Don't throw exception if queue is finished --- src/Coordination/KeeperStateMachine.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Coordination/KeeperStateMachine.cpp b/src/Coordination/KeeperStateMachine.cpp index dde8b30bf79..f76872bdd2f 100644 --- a/src/Coordination/KeeperStateMachine.cpp +++ b/src/Coordination/KeeperStateMachine.cpp @@ -248,7 +248,7 @@ nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, n session_id = storage->getSessionID(session_id_request.session_timeout_ms); LOG_DEBUG(log, "Session ID response {} with timeout {}", session_id, session_id_request.session_timeout_ms); response->session_id = session_id; - if (!responses_queue.push(response_for_session)) + if (!responses_queue.push(response_for_session) && !responses_queue.isFinished()) { ProfileEvents::increment(ProfileEvents::KeeperCommitsFailed); throw Exception(ErrorCodes::SYSTEM_ERROR, "Could not push response with session id {} into responses queue", session_id); @@ -261,7 +261,7 @@ nuraft::ptr KeeperStateMachine::commit(const uint64_t log_idx, n KeeperStorage::ResponsesForSessions responses_for_sessions = storage->processRequest( request_for_session.request, request_for_session.session_id, request_for_session.zxid); for (auto & response_for_session : responses_for_sessions) - if (!responses_queue.push(response_for_session)) + if (!responses_queue.push(response_for_session) && !responses_queue.isFinished()) { ProfileEvents::increment(ProfileEvents::KeeperCommitsFailed); throw Exception( @@ -523,7 +523,7 @@ void KeeperStateMachine::processReadRequest(const KeeperStorage::RequestForSessi true /*check_acl*/, true /*is_local*/); for (const auto & response : responses) - if (!responses_queue.push(response)) + if (!responses_queue.push(response) && !responses_queue.isFinished()) throw Exception( ErrorCodes::SYSTEM_ERROR, "Could not push response with session id {} into responses queue", response.session_id); } From d8d012523d536c9ac4def0524b6c0d32bc2935a2 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:47:50 +0000 Subject: [PATCH 56/88] add pre-pull to runner --- tests/integration/runner | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/tests/integration/runner b/tests/integration/runner index f4f853e00ad..975f5f21bd5 100755 --- a/tests/integration/runner +++ b/tests/integration/runner @@ -242,6 +242,14 @@ if __name__ == "__main__": "-n", "--parallel", action="store", dest="parallel", help="Parallelism" ) + parser.add_argument( + "--no-random", action="store", dest="no_random", help="Disable tests order randomization" + ) + + parser.add_argument( + "--pre-pull", action="store_true", default=False, dest="pre_pull", help="Pull images for docker_compose before all other actions" + ) + parser.add_argument( "-t", "--tests_list", @@ -374,7 +382,7 @@ if __name__ == "__main__": if args.keyword_expression: args.pytest_args += ["-k", args.keyword_expression] - cmd = "docker run {net} {tty} --rm --name {name} --privileged \ + cmd_base = "docker run {net} {tty} --rm --name {name} --privileged \ --volume={odbc_bridge_bin}:/clickhouse-odbc-bridge --volume={bin}:/clickhouse \ --volume={library_bridge_bin}:/clickhouse-library-bridge \ --volume={base_cfg}:/clickhouse-config --volume={cases_dir}:/ClickHouse/tests/integration \ @@ -383,7 +391,7 @@ if __name__ == "__main__": {dockerd_internal_volume} -e DOCKER_CLIENT_TIMEOUT=300 -e COMPOSE_HTTP_TIMEOUT=600 \ -e XTABLES_LOCKFILE=/run/host/xtables.lock \ -e PYTHONUNBUFFERED=1 \ - {env_tags} {env_cleanup} -e PYTEST_OPTS='{parallel} {opts} {tests_list} -vvv' {img} {command}".format( + {env_tags} {env_cleanup} -e PYTEST_OPTS='{parallel} {opts} {tests_list} -vvv' {img}".format( net=net, tty=tty, bin=args.binary, @@ -400,9 +408,11 @@ if __name__ == "__main__": dockerd_internal_volume=dockerd_internal_volume, img=DIND_INTEGRATION_TESTS_IMAGE_NAME + ":" + args.docker_image_version, name=CONTAINER_NAME, - command=args.command, ) + cmd = cmd_base + " " + args.command + cmd_pre_pull = cmd_base + " find /compose -name docker_compose_*.yml -exec docker-compose -f '{}' pull \;" + containers = subprocess.check_output( f"docker ps --all --quiet --filter name={CONTAINER_NAME} --format={{{{.ID}}}}", shell=True, @@ -413,5 +423,9 @@ if __name__ == "__main__": subprocess.check_call(f"docker kill {' '.join(containers)}", shell=True) print(f"Containers {containers} killed") + if args.pre_pull: + print(("Running pre pull as: '" + cmd_pre_pull + "'.")) + subprocess.check_call(cmd_pre_pull, shell=True) + print(("Running pytest container as: '" + cmd + "'.")) subprocess.check_call(cmd, shell=True) From 03582f4b8707bd16c46bf2b476a635adbcacfeca Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Jan 2023 15:52:34 +0000 Subject: [PATCH 57/88] move logic to runner --- tests/integration/ci-runner.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 73fed68ba89..264f6d843d2 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -297,11 +297,11 @@ class ClickhouseIntegrationTestsRunner: cmd = ( "cd {repo_path}/tests/integration && " - "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --command '{command}' ".format( + "timeout -s 9 1h ./runner {runner_opts} {image_cmd} --pre-pull --command '{command}' ".format( repo_path=repo_path, runner_opts=self._get_runner_opts(), image_cmd=image_cmd, - command=r""" find /compose -name docker_compose_*.yml -exec docker-compose -f {} pull \; """, + command=r""" echo Pre Pull finished """, ) ) From 8f21eb543c75b1baf8a4726a3be0bfcbdac8571b Mon Sep 17 00:00:00 2001 From: Dmitry Novik Date: Wed, 4 Jan 2023 16:00:47 +0000 Subject: [PATCH 58/88] Fix argument evaluation order --- src/Interpreters/ActionsDAG.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 6e935898ddd..94c59b49c5d 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -192,8 +192,9 @@ const ActionsDAG::Node & ActionsDAG::addFunction( { auto [arguments, all_const] = getFunctionArguments(children); + auto function_base = function->build(arguments); return addFunctionImpl( - function->build(arguments), + function_base, std::move(children), std::move(arguments), std::move(result_name), From 2e758f983fb86732e0467f6a9bc2deb266d51a40 Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Jan 2023 18:08:55 +0100 Subject: [PATCH 59/88] Add logging, remove closing connection in destructor --- src/Storages/RabbitMQ/RabbitMQConnection.cpp | 20 -------------------- src/Storages/RabbitMQ/RabbitMQConnection.h | 2 -- src/Storages/RabbitMQ/StorageRabbitMQ.cpp | 7 ++++++- 3 files changed, 6 insertions(+), 23 deletions(-) diff --git a/src/Storages/RabbitMQ/RabbitMQConnection.cpp b/src/Storages/RabbitMQ/RabbitMQConnection.cpp index 39aa2b87caf..13d065774a2 100644 --- a/src/Storages/RabbitMQ/RabbitMQConnection.cpp +++ b/src/Storages/RabbitMQ/RabbitMQConnection.cpp @@ -122,24 +122,4 @@ void RabbitMQConnection::disconnectImpl(bool immediately) event_handler.iterateLoop(); } -RabbitMQConnection::~RabbitMQConnection() -{ - std::lock_guard lock(mutex); - - if (!connection) - return; - - try - { - /// Try to always close the connection gracefully (run the loop to see the closing callbacks) - /// to make sure that the associated callbacks and pending events are removed - /// before handler and loop are destructed. - disconnectImpl(); - } - catch (...) - { - tryLogCurrentException(__PRETTY_FUNCTION__); - } -} - } diff --git a/src/Storages/RabbitMQ/RabbitMQConnection.h b/src/Storages/RabbitMQ/RabbitMQConnection.h index ac9380d25db..698230b16f4 100644 --- a/src/Storages/RabbitMQ/RabbitMQConnection.h +++ b/src/Storages/RabbitMQ/RabbitMQConnection.h @@ -24,8 +24,6 @@ class RabbitMQConnection public: RabbitMQConnection(const RabbitMQConfiguration & configuration_, Poco::Logger * log_); - ~RabbitMQConnection(); - bool isConnected(); bool connect(); diff --git a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp index 2a4549e79ca..04decb91f7d 100644 --- a/src/Storages/RabbitMQ/StorageRabbitMQ.cpp +++ b/src/Storages/RabbitMQ/StorageRabbitMQ.cpp @@ -621,7 +621,6 @@ void StorageRabbitMQ::prepareChannelForConsumer(RabbitMQConsumerPtr consumer) consumer->setupChannel(); } - void StorageRabbitMQ::unbindExchange() { /* This is needed because with RabbitMQ (without special adjustments) can't, for example, properly make mv if there was insert query @@ -812,6 +811,8 @@ void StorageRabbitMQ::shutdown() { shutdown_called = true; + LOG_TRACE(log, "Deactivating background tasks"); + /// In case it has not yet been able to setup connection; deactivateTask(connection_task, true, false); @@ -820,6 +821,8 @@ void StorageRabbitMQ::shutdown() deactivateTask(streaming_task, true, false); deactivateTask(looping_task, true, true); + LOG_TRACE(log, "Cleaning up RabbitMQ after table usage"); + /// Just a paranoid try catch, it is not actually needed. try { @@ -842,6 +845,8 @@ void StorageRabbitMQ::shutdown() { tryLogCurrentException(log); } + + LOG_TRACE(log, "Shutdown finished"); } From 533911b28b023c45e1f1ec1b4f86a3c8ba452afc Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Wed, 4 Jan 2023 19:59:15 +0000 Subject: [PATCH 60/88] Respect requested columns order for select. --- src/Interpreters/ExpressionAnalyzer.cpp | 13 +++++++- src/Interpreters/ExpressionAnalyzer.h | 4 +-- src/Interpreters/InterpreterSelectQuery.cpp | 2 +- .../InterpreterSelectWithUnionQuery.cpp | 4 ++- .../02517_union_columns_order.reference | 0 .../0_stateless/02517_union_columns_order.sql | 32 +++++++++++++++++++ 6 files changed, 50 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/02517_union_columns_order.reference create mode 100644 tests/queries/0_stateless/02517_union_columns_order.sql diff --git a/src/Interpreters/ExpressionAnalyzer.cpp b/src/Interpreters/ExpressionAnalyzer.cpp index a3db464fbbb..1079ed3f714 100644 --- a/src/Interpreters/ExpressionAnalyzer.cpp +++ b/src/Interpreters/ExpressionAnalyzer.cpp @@ -1668,12 +1668,14 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio ExpressionActionsChain::Step & step = chain.lastStep(aggregated_columns); NamesWithAliases result_columns; + NameSet required_result_columns_set(required_result_columns.begin(), required_result_columns.end()); ASTs asts = select_query->select()->children; for (const auto & ast : asts) { String result_name = ast->getAliasOrColumnName(); - if (required_result_columns.empty() || required_result_columns.contains(result_name)) + + if (required_result_columns_set.empty() || required_result_columns_set.contains(result_name)) { std::string source_name = ast->getColumnName(); @@ -1709,6 +1711,15 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendProjectResult(ExpressionActio auto actions = chain.getLastActions(); actions->project(result_columns); + + if (!required_result_columns.empty()) + { + result_columns.clear(); + for (const auto & column : required_result_columns) + result_columns.emplace_back(column, std::string{}); + actions->project(result_columns); + } + return actions; } diff --git a/src/Interpreters/ExpressionAnalyzer.h b/src/Interpreters/ExpressionAnalyzer.h index ddb41a00f84..3d6261fe7d1 100644 --- a/src/Interpreters/ExpressionAnalyzer.h +++ b/src/Interpreters/ExpressionAnalyzer.h @@ -307,7 +307,7 @@ public: const TreeRewriterResultPtr & syntax_analyzer_result_, ContextPtr context_, const StorageMetadataPtr & metadata_snapshot_, - const NameSet & required_result_columns_ = {}, + const Names & required_result_columns_ = {}, bool do_global_ = false, const SelectQueryOptions & options_ = {}, PreparedSetsPtr prepared_sets_ = nullptr) @@ -364,7 +364,7 @@ public: private: StorageMetadataPtr metadata_snapshot; /// If non-empty, ignore all expressions not from this list. - NameSet required_result_columns; + Names required_result_columns; SelectQueryOptions query_options; JoinPtr makeJoin( diff --git a/src/Interpreters/InterpreterSelectQuery.cpp b/src/Interpreters/InterpreterSelectQuery.cpp index 9111cad3e16..64999b3ff6e 100644 --- a/src/Interpreters/InterpreterSelectQuery.cpp +++ b/src/Interpreters/InterpreterSelectQuery.cpp @@ -568,7 +568,7 @@ InterpreterSelectQuery::InterpreterSelectQuery( syntax_analyzer_result, context, metadata_snapshot, - NameSet(required_result_column_names.begin(), required_result_column_names.end()), + required_result_column_names, !options.only_analyze, options, prepared_sets); diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index c695dae6d53..40d7c79aa33 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -165,7 +165,9 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( for (size_t query_num = 0; query_num < num_children; ++query_num) { headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); - const auto & current_required_result_column_names = required_result_column_names_for_other_selects[query_num]; + const auto & current_required_result_column_names = (query_num == 0 && !require_full_header) + ? required_result_column_names + : required_result_column_names_for_other_selects[query_num]; if (!current_required_result_column_names.empty()) { const auto & header_columns = headers[query_num].getNames(); diff --git a/tests/queries/0_stateless/02517_union_columns_order.reference b/tests/queries/0_stateless/02517_union_columns_order.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02517_union_columns_order.sql b/tests/queries/0_stateless/02517_union_columns_order.sql new file mode 100644 index 00000000000..c02dacfa0df --- /dev/null +++ b/tests/queries/0_stateless/02517_union_columns_order.sql @@ -0,0 +1,32 @@ +CREATE TABLE t1 (c0 Int32, PRIMARY KEY (c0)) ENGINE = MergeTree; +SELECT DISTINCT * +FROM +( + SELECT DISTINCT + cos(sign(exp(t1.c0))), + -min2(pow(t1.c0, t1.c0), intDiv(t1.c0, t1.c0)), + t1.c0, + t1.c0, + erf(abs(-t1.c0)) + FROM t1 + WHERE t1.c0 > 0 + UNION ALL + SELECT DISTINCT + cos(sign(exp(t1.c0))), + -min2(pow(t1.c0, t1.c0), intDiv(t1.c0, t1.c0)), + t1.c0, + t1.c0, + erf(abs(-t1.c0)) + FROM t1 + WHERE NOT (t1.c0 > 0) + UNION ALL + SELECT DISTINCT + cos(sign(exp(t1.c0))), + -min2(pow(t1.c0, t1.c0), intDiv(t1.c0, t1.c0)), + t1.c0, + t1.c0, + erf(abs(-t1.c0)) + FROM t1 + WHERE t1.c0 > (0 IS NULL) +); + From ccb6226d22d3a97a4a318762014997490dee59dc Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Wed, 4 Jan 2023 20:07:52 +0000 Subject: [PATCH 61/88] install CH before test --- tests/integration/ci-runner.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 264f6d843d2..89e8d6a2cf2 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -833,6 +833,10 @@ class ClickhouseIntegrationTestsRunner: ) self._install_clickhouse(build_path) + + logging.info("Pulling images") + runner._pre_pull_images(repo_path) + logging.info( "Dump iptables before run %s", subprocess.check_output("sudo iptables -nvL", shell=True), @@ -1000,9 +1004,6 @@ if __name__ == "__main__": params = json.loads(open(params_path, "r").read()) runner = ClickhouseIntegrationTestsRunner(result_path, params) - logging.info("Pulling images") - runner._pre_pull_images(repo_path) - logging.info("Running tests") # Avoid overlaps with previous runs From ea7c338a7bed9d6c6c00614ed1f343b930188384 Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 4 Jan 2023 20:28:03 +0000 Subject: [PATCH 62/88] Done --- tests/integration/test_lost_part/test.py | 79 ++++++++++++++---------- 1 file changed, 46 insertions(+), 33 deletions(-) diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index 7a71044e5db..757292950ee 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -26,9 +26,7 @@ def start_cluster(): def remove_part_from_disk(node, table, part_name): part_path = node.query( - "SELECT path FROM system.parts WHERE table = '{}' and name = '{}'".format( - table, part_name - ) + f"SELECT path FROM system.parts WHERE table = '{table}' and name = '{part_name}'" ).strip() if not part_path: raise Exception("Part " + part_name + "doesn't exist") @@ -38,29 +36,31 @@ def remove_part_from_disk(node, table, part_name): def test_lost_part_same_replica(start_cluster): + node1.query("DROP TABLE IF EXISTS mt0 SYNC") + node2.query("DROP TABLE IF EXISTS mt0 SYNC") + + for node in [node1, node2]: node.query( - "CREATE TABLE mt0 (id UInt64, date Date) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{}') ORDER BY tuple() PARTITION BY date " - "SETTINGS cleanup_delay_period=1, cleanup_delay_period_random_add=1".format( - node.name - ) + f"CREATE TABLE mt0 (id UInt64, date Date) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{node.name}') ORDER BY tuple() PARTITION BY date " + "SETTINGS cleanup_delay_period=1, cleanup_delay_period_random_add=1" ) node1.query("SYSTEM STOP MERGES mt0") node2.query("SYSTEM STOP REPLICATION QUEUES") for i in range(5): - node1.query("INSERT INTO mt0 VALUES ({}, toDate('2020-10-01'))".format(i)) + node1.query(f"INSERT INTO mt0 VALUES ({i}, toDate('2020-10-01'))") for i in range(20): parts_to_merge = node1.query( - "SELECT parts_to_merge FROM system.replication_queue" + "SELECT parts_to_merge FROM system.replication_queue WHERE table='mt0' AND length(parts_to_merge) > 0" ) if parts_to_merge: parts_list = list(sorted(ast.literal_eval(parts_to_merge))) print("Got parts list", parts_list) if len(parts_list) < 3: - raise Exception("Got too small parts list {}".format(parts_list)) + raise Exception(f"Got too small parts list {parts_list}") break time.sleep(1) @@ -90,9 +90,7 @@ def test_lost_part_same_replica(start_cluster): assert node1.contains_in_log( "Created empty part" - ), "Seems like empty part {} is not created or log message changed".format( - victim_part_from_the_middle - ) + ), f"Seems like empty part {victim_part_from_the_middle} is not created or log message changed" assert node1.query("SELECT COUNT() FROM mt0") == "4\n" @@ -101,25 +99,29 @@ def test_lost_part_same_replica(start_cluster): assert_eq_with_retry(node2, "SELECT COUNT() FROM mt0", "4") assert_eq_with_retry(node2, "SELECT COUNT() FROM system.replication_queue", "0") + node1.query("DROP TABLE IF EXISTS mt0 SYNC") + node2.query("DROP TABLE IF EXISTS mt0 SYNC") + def test_lost_part_other_replica(start_cluster): + node1.query("DROP TABLE IF EXISTS mt1 SYNC") + node2.query("DROP TABLE IF EXISTS mt1 SYNC") + for node in [node1, node2]: node.query( - "CREATE TABLE mt1 (id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t1', '{}') ORDER BY tuple() " - "SETTINGS cleanup_delay_period=1, cleanup_delay_period_random_add=1".format( - node.name - ) + f"CREATE TABLE mt1 (id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t1', '{node.name}') ORDER BY tuple() " + "SETTINGS cleanup_delay_period=1, cleanup_delay_period_random_add=1" ) node1.query("SYSTEM STOP MERGES mt1") node2.query("SYSTEM STOP REPLICATION QUEUES") for i in range(5): - node1.query("INSERT INTO mt1 VALUES ({})".format(i)) + node1.query(f"INSERT INTO mt1 VALUES ({i})") for i in range(20): parts_to_merge = node1.query( - "SELECT parts_to_merge FROM system.replication_queue" + "SELECT parts_to_merge FROM system.replication_queue WHERE table='mt1' AND length(parts_to_merge) > 0" ) if parts_to_merge: parts_list = list(sorted(ast.literal_eval(parts_to_merge))) @@ -166,28 +168,32 @@ def test_lost_part_other_replica(start_cluster): assert_eq_with_retry(node1, "SELECT COUNT() FROM mt1", "4") assert_eq_with_retry(node1, "SELECT COUNT() FROM system.replication_queue", "0") + node1.query("DROP TABLE IF EXISTS mt1 SYNC") + node2.query("DROP TABLE IF EXISTS mt1 SYNC") + def test_lost_part_mutation(start_cluster): + node1.query("DROP TABLE IF EXISTS mt2 SYNC") + node2.query("DROP TABLE IF EXISTS mt2 SYNC") + for node in [node1, node2]: node.query( - "CREATE TABLE mt2 (id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t2', '{}') ORDER BY tuple() " - "SETTINGS cleanup_delay_period=1, cleanup_delay_period_random_add=1".format( - node.name - ) + f"CREATE TABLE mt2 (id UInt64) ENGINE ReplicatedMergeTree('/clickhouse/tables/t2', '{node.name}') ORDER BY tuple() " + "SETTINGS cleanup_delay_period=1, cleanup_delay_period_random_add=1" ) node1.query("SYSTEM STOP MERGES mt2") node2.query("SYSTEM STOP REPLICATION QUEUES") for i in range(2): - node1.query("INSERT INTO mt2 VALUES ({})".format(i)) + node1.query(f"INSERT INTO mt2 VALUES ({i})") node1.query( "ALTER TABLE mt2 UPDATE id = 777 WHERE 1", settings={"mutations_sync": "0"} ) for i in range(20): - parts_to_mutate = node1.query("SELECT count() FROM system.replication_queue") + parts_to_mutate = node1.query("SELECT count() FROM system.replication_queue WHERE table='mt2'") # two mutations for both replicas if int(parts_to_mutate) == 4: break @@ -223,21 +229,25 @@ def test_lost_part_mutation(start_cluster): assert_eq_with_retry(node2, "SELECT SUM(id) FROM mt2", "777") assert_eq_with_retry(node2, "SELECT COUNT() FROM system.replication_queue", "0") + node1.query("DROP TABLE IF EXISTS mt2 SYNC") + node2.query("DROP TABLE IF EXISTS mt2 SYNC") + def test_lost_last_part(start_cluster): + node1.query("DROP TABLE IF EXISTS mt3 SYNC") + node2.query("DROP TABLE IF EXISTS mt3 SYNC") + for node in [node1, node2]: node.query( - "CREATE TABLE mt3 (id UInt64, p String) ENGINE ReplicatedMergeTree('/clickhouse/tables/t3', '{}') " - "ORDER BY tuple() PARTITION BY p SETTINGS cleanup_delay_period=1, cleanup_delay_period_random_add=1".format( - node.name - ) + f"CREATE TABLE mt3 (id UInt64, p String) ENGINE ReplicatedMergeTree('/clickhouse/tables/t3', '{node.name}') " + "ORDER BY tuple() PARTITION BY p SETTINGS cleanup_delay_period=1, cleanup_delay_period_random_add=1" ) node1.query("SYSTEM STOP MERGES mt3") node2.query("SYSTEM STOP REPLICATION QUEUES") for i in range(1): - node1.query("INSERT INTO mt3 VALUES ({}, 'x')".format(i)) + node1.query(f"INSERT INTO mt3 VALUES ({i}, 'x')") # actually not important node1.query( @@ -245,7 +255,7 @@ def test_lost_last_part(start_cluster): ) partition_id = node1.query("select partitionId('x')").strip() - remove_part_from_disk(node1, "mt3", "{}_0_0_0".format(partition_id)) + remove_part_from_disk(node1, "mt3", f"{partition_id}_0_0_0") # other way to detect broken parts node1.query("CHECK TABLE mt3") @@ -253,7 +263,7 @@ def test_lost_last_part(start_cluster): node1.query("SYSTEM START MERGES mt3") for i in range(10): - result = node1.query("SELECT count() FROM system.replication_queue") + result = node1.query("SELECT count() FROM system.replication_queue WHERE table='mt3'") assert int(result) <= 2, "Have a lot of entries in queue {}".format( node1.query("SELECT * FROM system.replication_queue FORMAT Vertical") ) @@ -269,7 +279,10 @@ def test_lost_last_part(start_cluster): else: assert False, "Don't have required messages in node1 log" - node1.query("ALTER TABLE mt3 DROP PARTITION ID '{}'".format(partition_id)) + node1.query(f"ALTER TABLE mt3 DROP PARTITION ID '{partition_id}'") assert_eq_with_retry(node1, "SELECT COUNT() FROM mt3", "0") assert_eq_with_retry(node1, "SELECT COUNT() FROM system.replication_queue", "0") + + node1.query("DROP TABLE IF EXISTS mt3 SYNC") + node2.query("DROP TABLE IF EXISTS mt3 SYNC") From e3ca9fb3e94d885223918d35a1ab3509939601ea Mon Sep 17 00:00:00 2001 From: Nikita Mikhaylov Date: Wed, 4 Jan 2023 20:30:52 +0000 Subject: [PATCH 63/88] Blakc --- tests/integration/test_lost_part/test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_lost_part/test.py b/tests/integration/test_lost_part/test.py index 757292950ee..dd4c2105d55 100644 --- a/tests/integration/test_lost_part/test.py +++ b/tests/integration/test_lost_part/test.py @@ -39,7 +39,6 @@ def test_lost_part_same_replica(start_cluster): node1.query("DROP TABLE IF EXISTS mt0 SYNC") node2.query("DROP TABLE IF EXISTS mt0 SYNC") - for node in [node1, node2]: node.query( f"CREATE TABLE mt0 (id UInt64, date Date) ENGINE ReplicatedMergeTree('/clickhouse/tables/t', '{node.name}') ORDER BY tuple() PARTITION BY date " @@ -193,7 +192,9 @@ def test_lost_part_mutation(start_cluster): ) for i in range(20): - parts_to_mutate = node1.query("SELECT count() FROM system.replication_queue WHERE table='mt2'") + parts_to_mutate = node1.query( + "SELECT count() FROM system.replication_queue WHERE table='mt2'" + ) # two mutations for both replicas if int(parts_to_mutate) == 4: break @@ -263,7 +264,9 @@ def test_lost_last_part(start_cluster): node1.query("SYSTEM START MERGES mt3") for i in range(10): - result = node1.query("SELECT count() FROM system.replication_queue WHERE table='mt3'") + result = node1.query( + "SELECT count() FROM system.replication_queue WHERE table='mt3'" + ) assert int(result) <= 2, "Have a lot of entries in queue {}".format( node1.query("SELECT * FROM system.replication_queue FORMAT Vertical") ) From 932a2f9e0983bb732730960260203d4f3076225d Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 5 Jan 2023 09:12:56 +0000 Subject: [PATCH 64/88] revert minio changes --- .../runner/compose/docker_compose_minio.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker/test/integration/runner/compose/docker_compose_minio.yml b/docker/test/integration/runner/compose/docker_compose_minio.yml index 45f9ef3c454..3eaf891ff8e 100644 --- a/docker/test/integration/runner/compose/docker_compose_minio.yml +++ b/docker/test/integration/runner/compose/docker_compose_minio.yml @@ -2,17 +2,19 @@ version: '2.3' services: minio1: - image: minio/minio:RELEASE.2022-12-12T19-27-27Z + # Newer version of minio results in such errors: + # "AWSErrorMarshaller: Encountered AWSError 'InternalError': We encountered an internal error, please try again" + image: minio/minio:RELEASE.2021-09-23T04-46-24Z volumes: - - ${MINIO_DATA_DIR}:/data1 + - data1-1:/data1 - ${MINIO_CERTS_DIR:-}:/certs expose: - ${MINIO_PORT:-9001} environment: - MINIO_ROOT_USER: minio - MINIO_ROOT_PASSWORD: minio123 + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: minio123 MINIO_PROMETHEUS_AUTH_TYPE: public - command: server --address :9001 --certs-dir /certs /data1 + command: server --address :9001 --certs-dir /certs /data1-1 depends_on: - proxy1 - proxy2 From fcb042d80c47b24a68fa47f45910657d34b59c8e Mon Sep 17 00:00:00 2001 From: kssenii Date: Wed, 4 Jan 2023 15:29:06 +0100 Subject: [PATCH 65/88] Fix cland tidy --- src/Storages/NamedCollectionsHelpers.cpp | 2 +- src/Storages/StorageURL.cpp | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Storages/NamedCollectionsHelpers.cpp b/src/Storages/NamedCollectionsHelpers.cpp index 2cb3f28ad42..cefed555781 100644 --- a/src/Storages/NamedCollectionsHelpers.cpp +++ b/src/Storages/NamedCollectionsHelpers.cpp @@ -67,7 +67,7 @@ NamedCollectionPtr tryGetNamedCollectionWithOverrides(ASTs asts) auto collection_copy = collection->duplicate(); - for (auto it = std::next(asts.begin()); it != asts.end(); ++it) + for (auto * it = std::next(asts.begin()); it != asts.end(); ++it) { auto value_override = getKeyValueFromAST(*it); if (!value_override && !(*it)->as()) diff --git a/src/Storages/StorageURL.cpp b/src/Storages/StorageURL.cpp index c66c1e61d33..dbe99d0ce9a 100644 --- a/src/Storages/StorageURL.cpp +++ b/src/Storages/StorageURL.cpp @@ -73,8 +73,8 @@ static const std::unordered_set optional_configuration_keys = /// Headers in config file will have structure "headers.header.name" and "headers.header.value". /// But Poco::AbstractConfiguration converts them into "header", "header[1]", "header[2]". static const std::vector optional_regex_keys = { - std::regex("headers.header\\[[\\d]*\\].name"), - std::regex("headers.header\\[[\\d]*\\].value"), + std::regex(R"(headers.header\[[\d]*\].name)"), + std::regex(R"(headers.header\[[\d]*\].value)"), }; static bool urlWithGlobs(const String & uri) From db46267a9503c9231725ba22b3703202cc82bd93 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Wed, 4 Jan 2023 17:06:46 +0000 Subject: [PATCH 66/88] Fast fix: force upper bound for time to delay INSERT --- src/Storages/MergeTree/MergeTreeData.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 358d527ae28..218a1cdf1b2 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3781,7 +3781,11 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, ContextPtr q max_k = settings->inactive_parts_to_throw_insert - settings->inactive_parts_to_delay_insert; k = k_inactive + 1; } - const UInt64 delay_milliseconds = static_cast(::pow(settings->max_delay_to_insert * 1000, static_cast(k) / max_k)); + + const UInt64 max_delay_milliseconds = (settings->max_delay_to_insert > 0 ? settings->max_delay_to_insert * 1000 : 1000); + /// max() as a save guard here + const UInt64 delay_milliseconds + = std::max(max_delay_milliseconds, static_cast(::pow(max_delay_milliseconds, static_cast(k) / max_k))); ProfileEvents::increment(ProfileEvents::DelayedInserts); ProfileEvents::increment(ProfileEvents::DelayedInsertsMilliseconds, delay_milliseconds); From 2a105d186201ffaac666581d5a98999ddeb7b4b4 Mon Sep 17 00:00:00 2001 From: Igor Nikonov Date: Thu, 5 Jan 2023 12:37:25 +0000 Subject: [PATCH 67/88] Fix stupid mistake - max() -> min() --- src/Storages/MergeTree/MergeTreeData.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 218a1cdf1b2..30d0570ff11 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -3783,9 +3783,9 @@ void MergeTreeData::delayInsertOrThrowIfNeeded(Poco::Event * until, ContextPtr q } const UInt64 max_delay_milliseconds = (settings->max_delay_to_insert > 0 ? settings->max_delay_to_insert * 1000 : 1000); - /// max() as a save guard here + /// min() as a save guard here const UInt64 delay_milliseconds - = std::max(max_delay_milliseconds, static_cast(::pow(max_delay_milliseconds, static_cast(k) / max_k))); + = std::min(max_delay_milliseconds, static_cast(::pow(max_delay_milliseconds, static_cast(k) / max_k))); ProfileEvents::increment(ProfileEvents::DelayedInserts); ProfileEvents::increment(ProfileEvents::DelayedInsertsMilliseconds, delay_milliseconds); From 9fd56e100a0479df3075b57fe5858b9541f54afe Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 5 Jan 2023 13:20:47 +0000 Subject: [PATCH 68/88] Add a comment. --- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 40d7c79aa33..58a6d98b8ae 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -165,6 +165,10 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( for (size_t query_num = 0; query_num < num_children; ++query_num) { headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); + /// Here whe check that, in case if required_result_column_names were specified, + /// nested interpreter returns exaclty it. Except if query requires full header. + /// The code aboew is written in a way that for 0th query required_result_column_names_for_other_selects[0] + /// is an empty list, and we should use required_result_column_names instead. const auto & current_required_result_column_names = (query_num == 0 && !require_full_header) ? required_result_column_names : required_result_column_names_for_other_selects[query_num]; From 1b03a60e436a39bb691d6f60a4c350a0b789a656 Mon Sep 17 00:00:00 2001 From: Yatsishin Ilya <2159081+qoega@users.noreply.github.com> Date: Thu, 5 Jan 2023 13:53:45 +0000 Subject: [PATCH 69/88] Do not report status if we failed to pull images. --- tests/ci/integration_test_check.py | 5 +++++ tests/integration/ci-runner.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tests/ci/integration_test_check.py b/tests/ci/integration_test_check.py index 86f38a5b8b4..a6935e22091 100644 --- a/tests/ci/integration_test_check.py +++ b/tests/ci/integration_test_check.py @@ -247,6 +247,11 @@ if __name__ == "__main__": retcode = process.wait() if retcode == 0: logging.info("Run tests successfully") + elif retcode == 13: + logging.warning( + "There were issues with infrastructure. Not writing status report to restart job." + ) + sys.exit(1) else: logging.info("Some tests failed") diff --git a/tests/integration/ci-runner.py b/tests/integration/ci-runner.py index 89e8d6a2cf2..4bb830351ce 100755 --- a/tests/integration/ci-runner.py +++ b/tests/integration/ci-runner.py @@ -317,7 +317,8 @@ class ClickhouseIntegrationTestsRunner: logging.info("docker-compose pull failed: " + str(err)) continue logging.error("Pulling images failed for 5 attempts. Will fail the worker.") - exit(1) + # We pass specific retcode to to ci/integration_test_check.py to skip status reporting and restart job + exit(13) def _can_run_with(self, path, opt): with open(path, "r") as script: From dc5009d4017475e27e0605a70d9baea89ab33303 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 5 Jan 2023 15:28:56 +0000 Subject: [PATCH 70/88] Stop merges to aviod a race between merge and freeze. --- tests/integration/test_multiple_disks/test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index 9b7bad2b256..ee87edc84d9 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -1618,6 +1618,8 @@ def test_rename(start_cluster): """ ) + node1.query("SYSTEM STOP MERGES default.renaming_table") + for _ in range(5): data = [] for i in range(10): From 273ad90d1d0f3b01adb4944c907c1423695ea98d Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Thu, 5 Jan 2023 10:47:18 -0500 Subject: [PATCH 71/88] add additional table filters setting --- docs/en/operations/settings/settings.md | 33 +++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/docs/en/operations/settings/settings.md b/docs/en/operations/settings/settings.md index 895e071a560..c2afd740ff2 100644 --- a/docs/en/operations/settings/settings.md +++ b/docs/en/operations/settings/settings.md @@ -6,6 +6,39 @@ slug: /en/operations/settings/settings # Settings +## additional_table_filters + +An additional filter expression that is applied after reading +from the specified table. + +Default value: 0. + +**Example** + +``` sql +insert into table_1 values (1, 'a'), (2, 'bb'), (3, 'ccc'), (4, 'dddd'); +``` +```response +┌─x─┬─y────┐ +│ 1 │ a │ +│ 2 │ bb │ +│ 3 │ ccc │ +│ 4 │ dddd │ +└───┴──────┘ +``` +```sql +SELECT * +FROM table_1 +SETTINGS additional_table_filters = (('table_1', 'x != 2')) +``` +```response +┌─x─┬─y────┐ +│ 1 │ a │ +│ 3 │ ccc │ +│ 4 │ dddd │ +└───┴──────┘ +``` + ## allow_nondeterministic_mutations {#allow_nondeterministic_mutations} User-level setting that allows mutations on replicated tables to make use of non-deterministic functions such as `dictGet`. From 4cba7fc9580a0d97458e1c314aab898bdef54bb3 Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Thu, 5 Jan 2023 11:10:20 -0500 Subject: [PATCH 72/88] add aliases for positiveModulo to docs --- .../functions/arithmetic-functions.md | 23 ++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/arithmetic-functions.md b/docs/en/sql-reference/functions/arithmetic-functions.md index 56f3a88b28b..e671b4b7166 100644 --- a/docs/en/sql-reference/functions/arithmetic-functions.md +++ b/docs/en/sql-reference/functions/arithmetic-functions.md @@ -65,10 +65,27 @@ An exception is thrown when dividing by zero or when dividing a minimal negative Differs from [modulo](#modulo) in that it returns zero when the divisor is zero. -## positive_modulo(a, b) -Calculates the remainder when dividing `a` by `b`. Similar to function `modulo` except that `positive_modulo` always return non-negative number. +## positiveModulo(a, b), positive_modulo(a, b), pmod(a, b) +Calculates the remainder when dividing `a` by `b`. Similar to the function `modulo` except that `positive_modulo` always returns a non-negative number. -Notice that `positive_modulo` is 4-5 times slower than `modulo`. You should not use `positive_modulo` unless you want to get positive result and don't care about performance too much. +Notice that `positive_modulo` is 4-5 times slower than `modulo`. You should not use `positive_modulo` unless you want to get a positive result and don't care about performance too much. + +**Example** + +Query: + +```sql +SELECT positiveModulo(-1, 10) +``` + +Result: + +```text + +┌─positiveModulo(-1, 10)─┐ +│ 9 │ +└────────────────────────┘ +``` ## negate(a), -a operator From 90f3f5789f8ebbd482f8a766d6c97b2f6686fa7e Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 5 Jan 2023 16:23:49 +0000 Subject: [PATCH 73/88] Speedup a bit. --- tests/integration/test_multiple_disks/test.py | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index ee87edc84d9..7d426355b43 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -407,10 +407,7 @@ def test_alter_policy(start_cluster, name, engine): def get_random_string(length): - return "".join( - random.choice(string.ascii_uppercase + string.digits) for _ in range(length) - ) - + return "randomPrintableASCII({})".format(length) def get_used_disks_for_table(node, table_name): return tuple( @@ -554,7 +551,7 @@ def test_max_data_part_size(start_cluster, name, engine): node1.query_with_retry( "INSERT INTO {} VALUES {}".format( - name, ",".join(["('" + x + "')" for x in data]) + name, ",".join(["(" + x + ")" for x in data]) ) ) used_disks = get_used_disks_for_table(node1, name) @@ -598,7 +595,7 @@ def test_jbod_overflow(start_cluster, name, engine): data.append(get_random_string(1024 * 1024)) # 1MB row node1.query_with_retry( "INSERT INTO {} VALUES {}".format( - name, ",".join(["('" + x + "')" for x in data]) + name, ",".join(["(" + x + ")" for x in data]) ) ) @@ -612,7 +609,7 @@ def test_jbod_overflow(start_cluster, name, engine): node1.query_with_retry( "INSERT INTO {} VALUES {}".format( - name, ",".join(["('" + x + "')" for x in data]) + name, ",".join(["(" + x + ")" for x in data]) ) ) @@ -676,7 +673,7 @@ def test_background_move(start_cluster, name, engine): # small jbod size is 40MB, so lets insert 5MB batch 5 times node1.query_with_retry( "INSERT INTO {} VALUES {}".format( - name, ",".join(["('" + x + "')" for x in data]) + name, ",".join(["(" + x + ")" for x in data]) ) ) @@ -694,8 +691,9 @@ def test_background_move(start_cluster, name, engine): # first (oldest) part was moved to external assert used_disks[0] == "external" + node1.query("SYSTEM FLUSH LOGS") path = node1.query( - "SELECT path_on_disk FROM system.part_log WHERE table = '{}' AND event_type='MovePart' ORDER BY event_time LIMIT 1".format( + "SELECT path_on_disk FROM system.part_log WHERE table = '{}' AND event_type='MovePart' AND part_name = 'all_1_1_0'".format( name ) ) @@ -788,7 +786,7 @@ def test_start_stop_moves(start_cluster, name, engine): # jbod size is 40MB, so lets insert 5MB batch 7 times node1.query_with_retry( "INSERT INTO {} VALUES {}".format( - name, ",".join(["('" + x + "')" for x in data]) + name, ",".join(["(" + x + ")" for x in data]) ) ) @@ -1302,7 +1300,7 @@ def test_detach_attach(start_cluster, name, engine): data.append(get_random_string(1024 * 1024)) # 1MB row node1.query_with_retry( "INSERT INTO {} VALUES {}".format( - name, ",".join(["('" + x + "')" for x in data]) + name, ",".join(["(" + x + ")" for x in data]) ) ) @@ -1354,7 +1352,7 @@ def test_mutate_to_another_disk(start_cluster, name, engine): data.append(get_random_string(1024 * 1024)) # 1MB row node1.query_with_retry( "INSERT INTO {} VALUES {}".format( - name, ",".join(["('" + x + "')" for x in data]) + name, ",".join(["(" + x + ")" for x in data]) ) ) @@ -1503,7 +1501,7 @@ def test_simple_replication_and_moves(start_cluster): data.append(get_random_string(512 * 1024)) # 500KB value node.query_with_retry( "INSERT INTO replicated_table_for_moves VALUES {}".format( - ",".join(["('" + x + "')" for x in data]) + ",".join(["(" + x + ")" for x in data]) ) ) @@ -1538,12 +1536,12 @@ def test_simple_replication_and_moves(start_cluster): node1.query_with_retry( "INSERT INTO replicated_table_for_moves VALUES {}".format( - ",".join(["('" + x + "')" for x in data]) + ",".join(["(" + x + ")" for x in data]) ) ) node2.query_with_retry( "INSERT INTO replicated_table_for_moves VALUES {}".format( - ",".join(["('" + x + "')" for x in data]) + ",".join(["(" + x + ")" for x in data]) ) ) @@ -1581,7 +1579,7 @@ def test_download_appropriate_disk(start_cluster): data.append(get_random_string(1024 * 1024)) # 1MB value node1.query_with_retry( "INSERT INTO replicated_table_for_download VALUES {}".format( - ",".join(["('" + x + "')" for x in data]) + ",".join(["(" + x + ")" for x in data]) ) ) @@ -1618,15 +1616,13 @@ def test_rename(start_cluster): """ ) - node1.query("SYSTEM STOP MERGES default.renaming_table") - for _ in range(5): data = [] for i in range(10): data.append(get_random_string(1024 * 1024)) # 1MB value node1.query( "INSERT INTO renaming_table VALUES {}".format( - ",".join(["('" + x + "')" for x in data]) + ",".join(["(" + x + ")" for x in data]) ) ) @@ -1667,6 +1663,8 @@ def test_freeze(start_cluster): """ ) + node1.query("SYSTEM STOP MERGES default.freezing_table") + for _ in range(5): data = [] dates = [] @@ -1675,7 +1673,7 @@ def test_freeze(start_cluster): dates.append("toDate('2019-03-05')") node1.query( "INSERT INTO freezing_table VALUES {}".format( - ",".join(["(" + d + ", '" + s + "')" for d, s in zip(dates, data)]) + ",".join(["(" + d + ", " + s + ")" for d, s in zip(dates, data)]) ) ) @@ -1719,7 +1717,7 @@ def test_kill_while_insert(start_cluster): data.append(get_random_string(1024 * 1024)) # 1MB value node1.query( "INSERT INTO {name} VALUES {}".format( - ",".join(["('" + s + "')" for s in data]), name=name + ",".join(["(" + s + ")" for s in data]), name=name ) ) From 1bbf8c84a4cb9fb955dcba7b6215fec991b27008 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Thu, 5 Jan 2023 17:29:16 +0100 Subject: [PATCH 74/88] Update InterpreterSelectWithUnionQuery.cpp --- src/Interpreters/InterpreterSelectWithUnionQuery.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp index 58a6d98b8ae..3bfd8e3bfe0 100644 --- a/src/Interpreters/InterpreterSelectWithUnionQuery.cpp +++ b/src/Interpreters/InterpreterSelectWithUnionQuery.cpp @@ -165,8 +165,8 @@ InterpreterSelectWithUnionQuery::InterpreterSelectWithUnionQuery( for (size_t query_num = 0; query_num < num_children; ++query_num) { headers[query_num] = nested_interpreters[query_num]->getSampleBlock(); - /// Here whe check that, in case if required_result_column_names were specified, - /// nested interpreter returns exaclty it. Except if query requires full header. + /// Here we check that, in case if required_result_column_names were specified, + /// nested interpreter returns exactly it. Except if query requires full header. /// The code aboew is written in a way that for 0th query required_result_column_names_for_other_selects[0] /// is an empty list, and we should use required_result_column_names instead. const auto & current_required_result_column_names = (query_num == 0 && !require_full_header) From 62a9b33bbb4b6a4f14116e9d395f85fbb3564f27 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Jan 2023 16:29:58 +0000 Subject: [PATCH 75/88] Automatic style fix --- tests/integration/test_multiple_disks/test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/integration/test_multiple_disks/test.py b/tests/integration/test_multiple_disks/test.py index 7d426355b43..0de9b22b8f8 100644 --- a/tests/integration/test_multiple_disks/test.py +++ b/tests/integration/test_multiple_disks/test.py @@ -409,6 +409,7 @@ def test_alter_policy(start_cluster, name, engine): def get_random_string(length): return "randomPrintableASCII({})".format(length) + def get_used_disks_for_table(node, table_name): return tuple( node.query( From 9e5c8acd92724a12744c77a0aa3c564917873e7a Mon Sep 17 00:00:00 2001 From: DanRoscigno Date: Thu, 5 Jan 2023 11:37:05 -0500 Subject: [PATCH 76/88] add toLastDayOfMonth to docs --- .../functions/date-time-functions.md | 31 ++++++++++++------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/docs/en/sql-reference/functions/date-time-functions.md b/docs/en/sql-reference/functions/date-time-functions.md index 89fa72de8bf..5f6fad9ab9a 100644 --- a/docs/en/sql-reference/functions/date-time-functions.md +++ b/docs/en/sql-reference/functions/date-time-functions.md @@ -302,16 +302,23 @@ Returns the date. The behavior of parsing incorrect dates is implementation specific. ClickHouse may return zero date, throw an exception or do “natural” overflow. ::: +## toLastDayOfMonth + +Rounds a date, or date with time, to the last day of the month. +Returns the date. + +Alias: `LAST_DAY`. + If `toLastDayOfMonth` is called with an argument of type `Date` greater then 2149-05-31, the result will be calculated from the argument 2149-05-31 instead. ## toMonday -Rounds down a date or date with time to the nearest Monday. +Rounds down a date, or date with time, to the nearest Monday. Returns the date. ## toStartOfWeek(t\[,mode\]) -Rounds down a date or date with time to the nearest Sunday or Monday by mode. +Rounds down a date, or date with time, to the nearest Sunday or Monday by mode. Returns the date. The mode argument works exactly like the mode argument to toWeek(). For the single-argument syntax, a mode value of 0 is used. @@ -410,43 +417,43 @@ Converts a date with time to a certain fixed date, while preserving the time. ## toRelativeYearNum -Converts a date or date with time to the number of the year, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the year, starting from a certain fixed point in the past. ## toRelativeQuarterNum -Converts a date or date with time to the number of the quarter, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the quarter, starting from a certain fixed point in the past. ## toRelativeMonthNum -Converts a date or date with time to the number of the month, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the month, starting from a certain fixed point in the past. ## toRelativeWeekNum -Converts a date or date with time to the number of the week, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the week, starting from a certain fixed point in the past. ## toRelativeDayNum -Converts a date or date with time to the number of the day, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the day, starting from a certain fixed point in the past. ## toRelativeHourNum -Converts a date or date with time to the number of the hour, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the hour, starting from a certain fixed point in the past. ## toRelativeMinuteNum -Converts a date or date with time to the number of the minute, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the minute, starting from a certain fixed point in the past. ## toRelativeSecondNum -Converts a date or date with time to the number of the second, starting from a certain fixed point in the past. +Converts a date, or date with time, to the number of the second, starting from a certain fixed point in the past. ## toISOYear -Converts a date or date with time to a UInt16 number containing the ISO Year number. +Converts a date, or date with time, to a UInt16 number containing the ISO Year number. ## toISOWeek -Converts a date or date with time to a UInt8 number containing the ISO Week number. +Converts a date, or date with time, to a UInt8 number containing the ISO Week number. ## toWeek(date\[,mode\]) From b0715ec5eb1fa00ecff8061a71fc38fdf11ae13d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Thu, 5 Jan 2023 18:47:02 +0100 Subject: [PATCH 77/88] Change error code on invalid background_pool_size config --- src/Storages/MergeTree/MergeTreeBackgroundExecutor.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h index ad50cd44189..5c1178a1bc1 100644 --- a/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h +++ b/src/Storages/MergeTree/MergeTreeBackgroundExecutor.h @@ -21,7 +21,7 @@ namespace DB { namespace ErrorCodes { - extern const int LOGICAL_ERROR; + extern const int INVALID_CONFIG_PARAMETER; } struct TaskRuntimeData; @@ -172,7 +172,7 @@ public: , metric(metric_) { if (max_tasks_count == 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Task count for MergeTreeBackgroundExecutor must not be zero"); + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Task count for MergeTreeBackgroundExecutor must not be zero"); pending.setCapacity(max_tasks_count); active.set_capacity(max_tasks_count); From 9ef7dac0b575680ed26afefaf581dac86e51a822 Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Thu, 5 Jan 2023 18:22:47 +0000 Subject: [PATCH 78/88] Fix exception fix in TraceCollector dtor Cf. #44758 Function try blocks in ctors/dtors implicitly rethrow, making them more or less useless. https://www.onlinegdb.com/pCKQA0wTIN (click RUN) https://stackoverflow.com/questions/5612486/when-is-a-function-try-block-useful/5612508#5612508 Fortunately, this seems the only place like that in the codebase. --- src/Interpreters/TraceCollector.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/Interpreters/TraceCollector.cpp b/src/Interpreters/TraceCollector.cpp index 40a5b1f228d..49588d490f5 100644 --- a/src/Interpreters/TraceCollector.cpp +++ b/src/Interpreters/TraceCollector.cpp @@ -31,18 +31,20 @@ TraceCollector::TraceCollector(std::shared_ptr trace_log_) TraceCollector::~TraceCollector() -try { - if (!thread.joinable()) - LOG_ERROR(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined"); - else - stop(); + try + { + if (!thread.joinable()) + LOG_ERROR(&Poco::Logger::get("TraceCollector"), "TraceCollector thread is malformed and cannot be joined"); + else + stop(); - TraceSender::pipe.close(); -} -catch (...) -{ - tryLogCurrentException("TraceCollector"); + TraceSender::pipe.close(); + } + catch (...) + { + tryLogCurrentException("TraceCollector"); + } } From a2a9c7dec5f71d75704ddde65c58b13031350015 Mon Sep 17 00:00:00 2001 From: Kseniia Sumarokova <54203879+kssenii@users.noreply.github.com> Date: Thu, 5 Jan 2023 21:35:52 +0100 Subject: [PATCH 79/88] Update TableFunctionURL.cpp --- src/TableFunctions/TableFunctionURL.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/TableFunctions/TableFunctionURL.cpp b/src/TableFunctions/TableFunctionURL.cpp index b4db03d4495..e7c8197a58c 100644 --- a/src/TableFunctions/TableFunctionURL.cpp +++ b/src/TableFunctions/TableFunctionURL.cpp @@ -30,7 +30,7 @@ void TableFunctionURL::parseArguments(const ASTPtr & ast, ContextPtr context) { const auto & ast_function = assert_cast(ast.get()); - auto & args = ast_function->children; + const auto & args = ast_function->children; if (args.empty()) throw Exception(ErrorCodes::BAD_ARGUMENTS, bad_arguments_error_message); From 1b31b8bb910915d9a37e74dd16af90165697e29f Mon Sep 17 00:00:00 2001 From: Alexander Gololobov <440544+davenger@users.noreply.github.com> Date: Fri, 6 Jan 2023 00:26:08 +0100 Subject: [PATCH 80/88] Do not check read result consistency when unwinding --- .../MergeTree/MergeTreeRangeReader.cpp | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/src/Storages/MergeTree/MergeTreeRangeReader.cpp b/src/Storages/MergeTree/MergeTreeRangeReader.cpp index ac5c3b1db2d..6f8da624449 100644 --- a/src/Storages/MergeTree/MergeTreeRangeReader.cpp +++ b/src/Storages/MergeTree/MergeTreeRangeReader.cpp @@ -516,11 +516,13 @@ void MergeTreeRangeReader::ReadResult::optimize(const FilterWithCachedCount & cu LOG_TEST(log, "ReadResult::optimize() before: {}", dumpInfo()); - SCOPE_EXIT(checkInternalConsistency()); - - SCOPE_EXIT({ - LOG_TEST(log, "ReadResult::optimize() after: {}", dumpInfo()); - }); + SCOPE_EXIT( + if (!std::uncaught_exceptions()) + { + checkInternalConsistency(); + LOG_TEST(log, "ReadResult::optimize() after: {}", dumpInfo()); + } + ); if (total_zero_rows_in_tails == filter.size()) { @@ -924,10 +926,11 @@ MergeTreeRangeReader::ReadResult MergeTreeRangeReader::read(size_t max_rows, Mar ReadResult read_result(log); - SCOPE_EXIT({ - LOG_TEST(log, "read() returned {}, sample block {}", - read_result.dumpInfo(), this->result_sample_block.dumpNames()); - }); + SCOPE_EXIT( + if (!std::uncaught_exceptions()) + LOG_TEST(log, "read() returned {}, sample block {}", + read_result.dumpInfo(), this->result_sample_block.dumpNames()); + ); if (prev_reader) { From 239734ceb6d26335420c72ce23e3068c305d52d6 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 6 Jan 2023 00:35:10 +0100 Subject: [PATCH 81/88] Fix memory leak in Aws::InitAPI. --- .gitmodules | 2 +- contrib/aws-s2n-tls | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 406e8a7e11e..5949f672fcc 100644 --- a/.gitmodules +++ b/.gitmodules @@ -323,4 +323,4 @@ url = https://github.com/awslabs/aws-c-compression.git [submodule "contrib/aws-s2n-tls"] path = contrib/aws-s2n-tls - url = https://github.com/aws/s2n-tls.git + url = https://github.com/ClickHouse/s2n-tls.git diff --git a/contrib/aws-s2n-tls b/contrib/aws-s2n-tls index 15d534e8a9c..0f1ba9e5c4a 160000 --- a/contrib/aws-s2n-tls +++ b/contrib/aws-s2n-tls @@ -1 +1 @@ -Subproject commit 15d534e8a9ca1eda6bacee514e37d08b4f38a526 +Subproject commit 0f1ba9e5c4a67cb3898de0c0b4f911d4194dc8de From 8a5a2f5b3d1ee5e468ff9b2a4ea0aaddf6397a66 Mon Sep 17 00:00:00 2001 From: Vitaly Baranov Date: Fri, 6 Jan 2023 00:45:47 +0100 Subject: [PATCH 82/88] Up the log level of tables dependencies graphs. --- src/Databases/TablesDependencyGraph.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Databases/TablesDependencyGraph.cpp b/src/Databases/TablesDependencyGraph.cpp index ee6ecb57eba..202f07f5bc0 100644 --- a/src/Databases/TablesDependencyGraph.cpp +++ b/src/Databases/TablesDependencyGraph.cpp @@ -715,7 +715,7 @@ void TablesDependencyGraph::log() const { if (nodes.empty()) { - LOG_TEST(getLogger(), "No tables"); + LOG_TRACE(getLogger(), "No tables"); return; } @@ -727,7 +727,7 @@ void TablesDependencyGraph::log() const String level_desc = (node->level == CYCLIC_LEVEL) ? "cyclic" : fmt::format("level {}", node->level); - LOG_TEST(getLogger(), "Table {} has {} ({})", node->storage_id, dependencies_desc, level_desc); + LOG_TRACE(getLogger(), "Table {} has {} ({})", node->storage_id, dependencies_desc, level_desc); } } From 1cc5bce1c53c71b587bf062927fd7b1603c017df Mon Sep 17 00:00:00 2001 From: Robert Schulze Date: Fri, 6 Jan 2023 11:14:49 +0000 Subject: [PATCH 83/88] Docs: Mention non-standard DOTALL behavior of ClickHouse's match() Cf. #34603 --- .../en/sql-reference/functions/string-search-functions.md | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docs/en/sql-reference/functions/string-search-functions.md b/docs/en/sql-reference/functions/string-search-functions.md index 048d0864863..f48354762e2 100644 --- a/docs/en/sql-reference/functions/string-search-functions.md +++ b/docs/en/sql-reference/functions/string-search-functions.md @@ -351,14 +351,16 @@ In all `multiSearch*` functions the number of needles should be less than 2 ## match(haystack, pattern) -Checks whether the string matches the regular expression `pattern` in `re2` syntax. `Re2` has a more limited [syntax](https://github.com/google/re2/wiki/Syntax) than Perl regular expressions. +Checks whether string `haystack` matches the regular expression `pattern`. The pattern is an [re2 regular expression](https://github.com/google/re2/wiki/Syntax) which has a more limited syntax than Perl regular expressions. -Returns 0 if it does not match, or 1 if it matches. +Returns 1 in case of a match, and 0 otherwise. Matching is based on UTF-8, e.g. `.` matches the Unicode code point `¥` which is represented in UTF-8 using two bytes. The regular expression must not contain null bytes. -If the haystack or pattern contain a sequence of bytes that are not valid UTF-8, then the behavior is undefined. +If the haystack or pattern contain a sequence of bytes that are not valid UTF-8, the behavior is undefined. No automatic Unicode normalization is performed, if you need it you can use the [normalizeUTF8*()](https://clickhouse.com/docs/en/sql-reference/functions/string-functions/) functions for that. +Unlike re2's default behavior, `.` matches line breaks. To disable this, prepend the pattern with `(?-s)`. + For patterns to search for substrings in a string, it is better to use LIKE or ‘position’, since they work much faster. ## multiMatchAny(haystack, \[pattern1, pattern2, …, patternn\]) From a0dad93a3864ece3dae65e203a9fdefcd83ee41c Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 6 Jan 2023 13:26:15 +0000 Subject: [PATCH 84/88] Check whatt if disable some checks in storage Merge. --- src/Storages/StorageMerge.cpp | 54 +++++++++++++++++------------------ src/Storages/StorageMerge.h | 4 +-- 2 files changed, 29 insertions(+), 29 deletions(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 028d6dd8caa..5ea7cab4741 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -831,7 +831,7 @@ void ReadFromMerge::convertingSourceStream( ContextPtr local_context, ASTPtr & query, QueryPipelineBuilder & builder, - QueryProcessingStage::Enum processed_stage) + QueryProcessingStage::Enum) { Block before_block_header = builder.getHeader(); @@ -874,33 +874,33 @@ void ReadFromMerge::convertingSourceStream( if (!where_expression) return; - if (processed_stage > QueryProcessingStage::FetchColumns) - { - for (size_t column_index : collections::range(0, header.columns())) - { - ColumnWithTypeAndName header_column = header.getByPosition(column_index); - ColumnWithTypeAndName before_column = before_block_header.getByName(header_column.name); - /// If the processed_stage greater than FetchColumns and the block structure between streams is different. - /// the where expression maybe invalid because of ConvertingTransform. - /// So we need to throw exception. - if (!header_column.type->equals(*before_column.type.get())) - { - NamesAndTypesList source_columns = metadata_snapshot->getSampleBlock().getNamesAndTypesList(); - auto virtual_column = *storage_merge->getVirtuals().tryGetByName("_table"); - source_columns.emplace_back(NameAndTypePair{virtual_column.name, virtual_column.type}); - auto syntax_result = TreeRewriter(local_context).analyze(where_expression, source_columns); - ExpressionActionsPtr actions = ExpressionAnalyzer{where_expression, syntax_result, local_context}.getActions(false, false); - Names required_columns = actions->getRequiredColumns(); + // if (processed_stage > QueryProcessingStage::FetchColumns) + // { + // for (size_t column_index : collections::range(0, header.columns())) + // { + // ColumnWithTypeAndName header_column = header.getByPosition(column_index); + // ColumnWithTypeAndName before_column = before_block_header.getByName(header_column.name); + // /// If the processed_stage greater than FetchColumns and the block structure between streams is different. + // /// the where expression maybe invalid because of ConvertingTransform. + // /// So we need to throw exception. + // if (!header_column.type->equals(*before_column.type.get())) + // { + // NamesAndTypesList source_columns = metadata_snapshot->getSampleBlock().getNamesAndTypesList(); + // auto virtual_column = *storage_merge->getVirtuals().tryGetByName("_table"); + // source_columns.emplace_back(NameAndTypePair{virtual_column.name, virtual_column.type}); + // auto syntax_result = TreeRewriter(local_context).analyze(where_expression, source_columns); + // ExpressionActionsPtr actions = ExpressionAnalyzer{where_expression, syntax_result, local_context}.getActions(false, false); + // Names required_columns = actions->getRequiredColumns(); - for (const auto & required_column : required_columns) - { - if (required_column == header_column.name) - throw Exception("Block structure mismatch in Merge Storage: different types:\n" + before_block_header.dumpStructure() - + "\n" + header.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - } - } - } - } + // for (const auto & required_column : required_columns) + // { + // if (required_column == header_column.name) + // throw Exception("Block structure mismatch in Merge Storage: different types:\n" + before_block_header.dumpStructure() + // + "\n" + header.dumpStructure(), ErrorCodes::LOGICAL_ERROR); + // } + // } + // } + // } } IStorage::ColumnSizeByName StorageMerge::getColumnSizes() const diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 54f3999514d..e42076eb594 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -193,10 +193,10 @@ private: size_t streams_num, bool concat_streams = false); - void convertingSourceStream( + static void convertingSourceStream( const Block & header, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, ContextPtr context, ASTPtr & query, - QueryPipelineBuilder & builder, QueryProcessingStage::Enum processed_stage); + QueryPipelineBuilder & builder, QueryProcessingStage::Enum); }; } From 71cb4ecde15fc80387cb9b57c17336620f242fdb Mon Sep 17 00:00:00 2001 From: Ilya Yatsishin <2159081+qoega@users.noreply.github.com> Date: Fri, 6 Jan 2023 15:30:57 +0100 Subject: [PATCH 85/88] Add docs for #43308 --- docs/en/operations/system-tables/data_skipping_indices.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/en/operations/system-tables/data_skipping_indices.md b/docs/en/operations/system-tables/data_skipping_indices.md index 338c6d02206..f1e233b33f7 100644 --- a/docs/en/operations/system-tables/data_skipping_indices.md +++ b/docs/en/operations/system-tables/data_skipping_indices.md @@ -11,6 +11,7 @@ Columns: - `table` ([String](../../sql-reference/data-types/string.md)) — Table name. - `name` ([String](../../sql-reference/data-types/string.md)) — Index name. - `type` ([String](../../sql-reference/data-types/string.md)) — Index type. +- `type_full` ([String](../../sql-reference/data-types/string.md)) — Index type expression from create statement. - `expr` ([String](../../sql-reference/data-types/string.md)) — Expression for the index calculation. - `granularity` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The number of granules in the block. - `data_compressed_bytes` ([UInt64](../../sql-reference/data-types/int-uint.md)) — The size of compressed data, in bytes. @@ -30,6 +31,7 @@ database: default table: user_actions name: clicks_idx type: minmax +type_full: minmax expr: clicks granularity: 1 data_compressed_bytes: 58 @@ -42,6 +44,7 @@ database: default table: users name: contacts_null_idx type: minmax +type_full: minmax expr: assumeNotNull(contacts_null) granularity: 1 data_compressed_bytes: 58 From 72eb74dd677c42044eae7f8470a6243d08d02ada Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 6 Jan 2023 14:50:28 +0000 Subject: [PATCH 86/88] Fix check for not existing input in ActionsDAG --- src/Interpreters/ActionsDAG.cpp | 8 +++++++- .../02521_cannot-find-column-in-projection.reference | 0 .../02521_cannot-find-column-in-projection.sql | 3 +++ 3 files changed, 10 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/02521_cannot-find-column-in-projection.reference create mode 100644 tests/queries/0_stateless/02521_cannot-find-column-in-projection.sql diff --git a/src/Interpreters/ActionsDAG.cpp b/src/Interpreters/ActionsDAG.cpp index 3b4d2dd1dd4..d9667800cf0 100644 --- a/src/Interpreters/ActionsDAG.cpp +++ b/src/Interpreters/ActionsDAG.cpp @@ -640,9 +640,15 @@ Block ActionsDAG::updateHeader(Block header) const arguments[i] = node_to_column[node->children[i]]; if (!arguments[i].column) throw Exception(ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK, - "Not found column {} in block", node->children[i]->result_name); + "Not found column {} in block {}", node->children[i]->result_name, + header.dumpStructure()); } + if (node->type == ActionsDAG::ActionType::INPUT) + throw Exception(ErrorCodes::NOT_FOUND_COLUMN_IN_BLOCK, + "Not found column {} in block {}", + node->result_name, header.dumpStructure()); + node_to_column[node] = executeActionForHeader(node, std::move(arguments)); } } diff --git a/tests/queries/0_stateless/02521_cannot-find-column-in-projection.reference b/tests/queries/0_stateless/02521_cannot-find-column-in-projection.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02521_cannot-find-column-in-projection.sql b/tests/queries/0_stateless/02521_cannot-find-column-in-projection.sql new file mode 100644 index 00000000000..31602c5bae2 --- /dev/null +++ b/tests/queries/0_stateless/02521_cannot-find-column-in-projection.sql @@ -0,0 +1,3 @@ +create table test(day Date, id UInt32) engine=MergeTree partition by day order by tuple(); +insert into test select toDate('2023-01-05') AS day, number from numbers(10); +with toUInt64(id) as id_with select day, count(id_with) from test where day >= '2023-01-01' group by day limit 1000; -- { serverError NOT_FOUND_COLUMN_IN_BLOCK } From 58c7547f517d5b831995b40331579904b1ffe270 Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 6 Jan 2023 15:04:00 +0000 Subject: [PATCH 87/88] Removed check and added a test. --- src/Storages/StorageMerge.cpp | 39 +------------------ src/Storages/StorageMerge.h | 4 +- ...different_types_in_storage_merge.reference | 0 ...02522_different_types_in_storage_merge.sql | 6 +++ 4 files changed, 10 insertions(+), 39 deletions(-) create mode 100644 tests/queries/0_stateless/02522_different_types_in_storage_merge.reference create mode 100644 tests/queries/0_stateless/02522_different_types_in_storage_merge.sql diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5ea7cab4741..5cfbb636ea7 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -653,7 +653,7 @@ QueryPipelineBuilderPtr ReadFromMerge::createSources( /// Subordinary tables could have different but convertible types, like numeric types of different width. /// We must return streams with structure equals to structure of Merge table. - convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, modified_query_info.query, *builder, processed_stage); + convertingSourceStream(header, storage_snapshot->metadata, aliases, modified_context, *builder); } return builder; @@ -829,9 +829,7 @@ void ReadFromMerge::convertingSourceStream( const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, ContextPtr local_context, - ASTPtr & query, - QueryPipelineBuilder & builder, - QueryProcessingStage::Enum) + QueryPipelineBuilder & builder) { Block before_block_header = builder.getHeader(); @@ -868,39 +866,6 @@ void ReadFromMerge::convertingSourceStream( return std::make_shared(stream_header, actions); }); } - - auto where_expression = query->as()->where(); - - if (!where_expression) - return; - - // if (processed_stage > QueryProcessingStage::FetchColumns) - // { - // for (size_t column_index : collections::range(0, header.columns())) - // { - // ColumnWithTypeAndName header_column = header.getByPosition(column_index); - // ColumnWithTypeAndName before_column = before_block_header.getByName(header_column.name); - // /// If the processed_stage greater than FetchColumns and the block structure between streams is different. - // /// the where expression maybe invalid because of ConvertingTransform. - // /// So we need to throw exception. - // if (!header_column.type->equals(*before_column.type.get())) - // { - // NamesAndTypesList source_columns = metadata_snapshot->getSampleBlock().getNamesAndTypesList(); - // auto virtual_column = *storage_merge->getVirtuals().tryGetByName("_table"); - // source_columns.emplace_back(NameAndTypePair{virtual_column.name, virtual_column.type}); - // auto syntax_result = TreeRewriter(local_context).analyze(where_expression, source_columns); - // ExpressionActionsPtr actions = ExpressionAnalyzer{where_expression, syntax_result, local_context}.getActions(false, false); - // Names required_columns = actions->getRequiredColumns(); - - // for (const auto & required_column : required_columns) - // { - // if (required_column == header_column.name) - // throw Exception("Block structure mismatch in Merge Storage: different types:\n" + before_block_header.dumpStructure() - // + "\n" + header.dumpStructure(), ErrorCodes::LOGICAL_ERROR); - // } - // } - // } - // } } IStorage::ColumnSizeByName StorageMerge::getColumnSizes() const diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index e42076eb594..d296eeea421 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -195,8 +195,8 @@ private: static void convertingSourceStream( const Block & header, const StorageMetadataPtr & metadata_snapshot, const Aliases & aliases, - ContextPtr context, ASTPtr & query, - QueryPipelineBuilder & builder, QueryProcessingStage::Enum); + ContextPtr context, + QueryPipelineBuilder & builder); }; } diff --git a/tests/queries/0_stateless/02522_different_types_in_storage_merge.reference b/tests/queries/0_stateless/02522_different_types_in_storage_merge.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02522_different_types_in_storage_merge.sql b/tests/queries/0_stateless/02522_different_types_in_storage_merge.sql new file mode 100644 index 00000000000..db0a498fd82 --- /dev/null +++ b/tests/queries/0_stateless/02522_different_types_in_storage_merge.sql @@ -0,0 +1,6 @@ +CREATE TABLE test_s64_local (date Date, value Int64) ENGINE = MergeTree order by tuple(); +CREATE TABLE test_u64_local (date Date, value UInt64) ENGINE = MergeTree order by tuple(); +CREATE TABLE test_s64_distributed AS test_s64_local ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_s64_local, rand()); +CREATE TABLE test_u64_distributed AS test_u64_local ENGINE = Distributed('test_shard_localhost', currentDatabase(), test_u64_local, rand()); + +SELECT * FROM merge(currentDatabase(), '') WHERE value = 1048575; From 398ae6a2164afcea498e3d16d43f02ed7665600f Mon Sep 17 00:00:00 2001 From: Nikolai Kochetov Date: Fri, 6 Jan 2023 15:33:19 +0000 Subject: [PATCH 88/88] Fix style. --- src/Storages/StorageMerge.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 5cfbb636ea7..e096811e5b9 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -44,7 +44,6 @@ namespace DB namespace ErrorCodes { extern const int BAD_ARGUMENTS; - extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; extern const int ILLEGAL_PREWHERE; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;