From 8629b29083308b58a036b83c8f20601534ac6c5a Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 12 May 2021 12:47:08 +0300 Subject: [PATCH 01/40] complain about unstable perf test queries We don't want to introduce more of them. The old ones should be all marked by now. --- docker/test/performance-comparison/report.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 9d3ccabb788..08164776ad8 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -555,9 +555,8 @@ if args.report == 'main': if unstable_queries: message_array.append(str(unstable_queries) + ' unstable') -# Disabled before fix. -# if very_unstable_queries: -# status = 'failure' + if very_unstable_queries: + status = 'failure' error_tests += slow_average_tests if error_tests: From d7b726045a89f95fa73ac7bbea29f603244439cf Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 13 May 2021 01:32:53 +0300 Subject: [PATCH 02/40] fixes --- docker/test/performance-comparison/report.py | 6 +++--- tests/performance/encodeXMLComponent.xml | 4 ++-- tests/performance/questdb_sum_int32.xml | 2 +- tests/performance/uniq.xml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 08164776ad8..c69162d1c77 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -552,11 +552,11 @@ if args.report == 'main': error_tests += unstable_partial_queries status = 'failure' - if unstable_queries: - message_array.append(str(unstable_queries) + ' unstable') - + # Don't show mildly unstable queries, only the very unstable ones we + # treat as errors. if very_unstable_queries: status = 'failure' + message_array.append(str(unstable_queries) + ' unstable') error_tests += slow_average_tests if error_tests: diff --git a/tests/performance/encodeXMLComponent.xml b/tests/performance/encodeXMLComponent.xml index 45241941ac3..7379a0ff89e 100644 --- a/tests/performance/encodeXMLComponent.xml +++ b/tests/performance/encodeXMLComponent.xml @@ -1,7 +1,7 @@ - test.hits + hits_100m_single - SELECT count() FROM test.hits WHERE NOT ignore(encodeXMLComponent(URL)) + SELECT count() FROM hits_100m_single WHERE NOT ignore(encodeXMLComponent(URL)) diff --git a/tests/performance/questdb_sum_int32.xml b/tests/performance/questdb_sum_int32.xml index 613ef3dc058..b1037aef5e2 100644 --- a/tests/performance/questdb_sum_int32.xml +++ b/tests/performance/questdb_sum_int32.xml @@ -1,4 +1,4 @@ - + 4 20G diff --git a/tests/performance/uniq.xml b/tests/performance/uniq.xml index b4e73733769..dd8d9154c0e 100644 --- a/tests/performance/uniq.xml +++ b/tests/performance/uniq.xml @@ -1,4 +1,4 @@ - + hits_100m_single From 3aa919b4988022d91fc3789584db4e1dced98c59 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 13 May 2021 12:16:43 +0300 Subject: [PATCH 03/40] fixes --- docker/test/performance-comparison/report.py | 3 +++ tests/performance/set_index.xml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index c69162d1c77..00aac5887fa 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -446,6 +446,9 @@ if args.report == 'main': attrs[3] = f'style="background: {color_bad}"' else: attrs[3] = '' + # Just don't add the slightly unstable queries we don't consider + # errors. It's not clear what the user should do with them. + continue text += tableRow(r, attrs, anchor) diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index 76f1087a1bf..03c249c26ff 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -62,7 +62,7 @@ - SELECT (rand(), rand()) IN tuple(tuple(17258, 93148), tuple(4508, 52749), tuple(68660, 70017), tuple(77797, 23528), tuple(1136, 37393), tuple(53237, 15379), tuple(68370, 73211), tuple(15782, 54962), tuple(59432, 45415), tuple(68396, 920), tuple(96154, 21016), tuple(12700, 26887), tuple(88016, 43191), tuple(68153, 51575), tuple(91315, 40005), tuple(18070, 73178), tuple(86, 631), tuple(77717, 20324), tuple(3227, 76188), tuple(74960, 43147), tuple(77538, 19628), tuple(82292, 6525), tuple(24293, 12566), tuple(85244, 96287), tuple(93982, 1329), tuple(38064, 54723), tuple(83999, 45810), tuple(71921, 53673), tuple(88638, 9669), tuple(1959, 39535), tuple(82235, 95796), tuple(27907, 90975), tuple(42383, 91015), tuple(9948, 91514), tuple(81712, 47309), tuple(400, 25808), tuple(31791, 46948), tuple(39740, 36098), tuple(25943, 84598), tuple(99598, 52939), tuple(77134, 15845), tuple(40313, 72174), tuple(85017, 94036), tuple(36595, 14303), tuple(83961, 68078), tuple(55792, 72759), tuple(73574, 43606), tuple(9853, 63560), tuple(28580, 56721), tuple(74804, 41025), tuple(32095, 55657), tuple(52881, 63416), tuple(91368, 90310), tuple(23922, 38883), tuple(30592, 10758), tuple(66448, 61183), tuple(31880, 96697), tuple(11362, 20633), tuple(75331, 2015), tuple(71129, 8785), tuple(1115, 70955), tuple(7886, 83698), tuple(18961, 84556), tuple(16677, 43028), tuple(37347, 70220), tuple(31699, 71244), tuple(10578, 96159), tuple(67600, 39041), tuple(78791, 86687), tuple(21545, 54174), tuple(68774, 37637), tuple(46132, 81768), tuple(98413, 20605), tuple(2960, 23665), tuple(31507, 35719), tuple(96209, 18368), tuple(60558, 38035), tuple(21952, 3264), tuple(11834, 86458), tuple(21651, 17650), tuple(86276, 36087), tuple(18818, 24849), tuple(61951, 3390), tuple(59637, 62545), tuple(30346, 72253), tuple(36281, 2992), tuple(78340, 49872), tuple(94326, 93723), tuple(3416, 94405), tuple(12272, 8741), tuple(22600, 22095), tuple(57636, 37106), tuple(38702, 14889), tuple(70238, 11276), tuple(17325, 60648), tuple(16492, 41271), tuple(52100, 1304), tuple(93416, 7795), tuple(57209, 71008), tuple(48010, 36078), tuple(20384, 74420), tuple(77440, 34439), tuple(69224, 45099), tuple(30374, 33884), tuple(49038, 90140), tuple(1154, 84725), tuple(64926, 86985), tuple(91746, 73472), tuple(59757, 75755), tuple(45860, 71557), tuple(45833, 36526), tuple(74618, 73598), tuple(91360, 65168), tuple(58029, 30793), tuple(56332, 14973), tuple(99943, 96877), tuple(97454, 6450), tuple(64502, 77301), tuple(73182, 31853), tuple(76809, 83964), tuple(82916, 86188), tuple(78736, 65427), tuple(36495, 7422), tuple(76196, 2804), tuple(96117, 61093), tuple(9177, 26099), tuple(52942, 63007), tuple(48578, 47876), tuple(50638, 89903), tuple(7113, 97316), tuple(35301, 12750), tuple(47807, 7254), tuple(38217, 55418), tuple(56970, 41687), tuple(20527, 62886), tuple(358, 14021), tuple(64018, 18582), tuple(91740, 21683), tuple(81967, 53589), tuple(45437, 38450), tuple(45476, 67752), tuple(76851, 72072), tuple(7304, 60091), tuple(40097, 12897), tuple(39906, 29247), tuple(84262, 58734), tuple(30857, 43791), tuple(56087, 78929), tuple(20498, 45954), tuple(48726, 500), tuple(62723, 43763), tuple(28368, 30756), tuple(74048, 52403), tuple(15045, 95926), tuple(75542, 55384), tuple(52543, 22525), tuple(56001, 6935), tuple(11431, 46745), tuple(77731, 7310), tuple(36718, 59909), tuple(32235, 91254), tuple(92417, 25917), tuple(21782, 79277), tuple(46378, 87536), tuple(35324, 26075), tuple(6310, 76915), tuple(1551, 69473), tuple(50642, 68865), tuple(55190, 72934), tuple(49780, 21873), tuple(99466, 29686), tuple(90761, 13179), tuple(72959, 57033), tuple(20020, 90200), tuple(46186, 79105), tuple(73871, 52382), tuple(59559, 38801), tuple(59916, 16082), tuple(33610, 94966), tuple(46001, 45225), tuple(86679, 26469), tuple(77245, 91929), tuple(32887, 36623), tuple(11179, 46898), tuple(87881, 68087), tuple(45438, 47991), tuple(24950, 94525), tuple(91664, 51656), tuple(43914, 47805), tuple(15736, 96156), tuple(56346, 20283), tuple(85053, 48931), tuple(17790, 26179), tuple(96195, 55728), tuple(43765, 54807), tuple(44988, 89269), tuple(55911, 99411), tuple(52446, 47397), tuple(28346, 65442), tuple(96669, 68226), tuple(66194, 26848), tuple(37276, 55864), tuple(14116, 41583), tuple(18058, 16317), tuple(93136, 85318), tuple(35616, 86252), tuple(29222, 29969), tuple(33386, 85372), tuple(71094, 44238), tuple(27733, 31838), tuple(64626, 16692), tuple(52904, 97899), tuple(97619, 12663), tuple(50165, 4688), tuple(67557, 44053), tuple(69184, 66269), tuple(73164, 89705), tuple(39822, 15169), tuple(65499, 72808), tuple(30068, 63697), tuple(30154, 64235), tuple(97016, 58716), tuple(94366, 36592), tuple(1592, 16261), tuple(87985, 52102), tuple(12554, 23652), tuple(15909, 25292), tuple(2527, 91531), tuple(92139, 36031), tuple(28986, 30032), tuple(3038, 56314), tuple(32239, 26707), tuple(15973, 34901), tuple(70246, 39680), tuple(82529, 38132), tuple(45827, 74783), tuple(53665, 64111), tuple(55218, 84170), tuple(20466, 16130), tuple(55734, 71203), tuple(31438, 96906), tuple(66338, 85858), tuple(35988, 68511), tuple(78391, 15191), tuple(80747, 59213), tuple(5357, 11546), tuple(16822, 16607), tuple(36607, 41106), tuple(74949, 30739), tuple(45726, 64887), tuple(1524, 54847), tuple(37371, 89195), tuple(28726, 27788), tuple(22600, 44777), tuple(53999, 63625), tuple(84304, 98338), tuple(49260, 76480), tuple(74564, 53907), tuple(89867, 97096), tuple(60157, 61299), tuple(17165, 10146), tuple(56334, 36268), tuple(62114, 49222), tuple(22715, 23620), tuple(42830, 11539), tuple(41091, 69151), tuple(75471, 68364), tuple(18681, 43249), tuple(42738, 63219), tuple(35474, 98454), tuple(76815, 46024), tuple(66310, 36521), tuple(86095, 77013), tuple(63693, 77319), tuple(80731, 63031), tuple(95478, 92387), tuple(23787, 63724), tuple(46299, 68994), tuple(4800, 2460), tuple(9663, 80639), tuple(77231, 85814), tuple(81615, 11311), tuple(35638, 27340), tuple(13598, 14322), tuple(30657, 17238), tuple(90957, 96846), tuple(69962, 52140), tuple(41681, 65962), tuple(96836, 58177), tuple(36190, 11623), tuple(4231, 40500), tuple(43049, 41949), tuple(71177, 98492), tuple(30193, 39750), tuple(19744, 33204), tuple(63358, 30210), tuple(45638, 58918), tuple(43641, 38741), tuple(35598, 40932), tuple(33238, 36236), tuple(50835, 20968), tuple(25099, 34071), tuple(84986, 88456), tuple(35333, 1529), tuple(79771, 23985), tuple(647, 61658), tuple(9424, 11743), tuple(77766, 31528), tuple(77811, 86973), tuple(76403, 74377), tuple(55568, 79251), tuple(68858, 20762), tuple(68520, 66773), tuple(93598, 89823), tuple(8080, 82539), tuple(87760, 52247), tuple(25191, 16905), tuple(17837, 8339), tuple(85177, 59050), tuple(51680, 77374), tuple(3287, 43018), tuple(43479, 62141), tuple(34909, 46322), tuple(11869, 5885), tuple(96193, 58417), tuple(101, 47460), tuple(34937, 88582), tuple(83216, 88388), tuple(28571, 15292), tuple(66683, 62613), tuple(34478, 8924), tuple(2680, 89973), tuple(62438, 44460), tuple(11724, 4791), tuple(5383, 72888), tuple(88206, 67586), tuple(8124, 21690), tuple(28779, 75789), tuple(66791, 4757), tuple(6176, 47760), tuple(6403, 78084), tuple(78122, 35446), tuple(99494, 73608), tuple(39691, 89098), tuple(59182, 19484), tuple(25389, 98963), tuple(96487, 3692), tuple(76222, 67381), tuple(21199, 50358), tuple(95998, 58137), tuple(28777, 43913), tuple(14176, 60117), tuple(52257, 81703), tuple(14604, 13438), tuple(71301, 14401), tuple(19758, 66914), tuple(15506, 29873), tuple(87205, 29449), tuple(93295, 15930), tuple(63651, 11287), tuple(19785, 15966), tuple(30795, 75112), tuple(69462, 37655), tuple(18793, 85764), tuple(36240, 31236), tuple(98153, 73724), tuple(72491, 4223), tuple(66930, 35048), tuple(25686, 13269), tuple(13940, 13259), tuple(69163, 11235), tuple(1183, 86961), tuple(54323, 67315), tuple(85044, 60872), tuple(48875, 3683), tuple(43052, 92861), tuple(87574, 32969), tuple(92552, 80564), tuple(94832, 47682), tuple(72011, 80994), tuple(60182, 917), tuple(97788, 34169), tuple(66432, 47940), tuple(87468, 80954), tuple(35385, 68758), tuple(50555, 63710), tuple(55311, 44337), tuple(87065, 26514), tuple(84581, 98736), tuple(23212, 56499), tuple(75120, 72447), tuple(56087, 38285), tuple(58171, 45629), tuple(28401, 44319), tuple(70432, 27883), tuple(18891, 14646), tuple(26206, 49924), tuple(79957, 44914), tuple(56064, 27529), tuple(99090, 29197), tuple(49435, 340), tuple(53525, 65601), tuple(76998, 88349), tuple(50416, 70860), tuple(42506, 75290), tuple(34024, 13295), tuple(86663, 46523), tuple(88814, 231), tuple(57809, 21), tuple(84914, 84771), tuple(43042, 66892), tuple(17288, 33908), tuple(4934, 63195), tuple(50590, 1516), tuple(97843, 80208), tuple(20091, 86717), tuple(71566, 15929), tuple(19531, 23634), tuple(41646, 45549), tuple(89226, 82902), tuple(96683, 63386), tuple(31072, 53788), tuple(51135, 41099), tuple(78912, 65609), tuple(36094, 23603), tuple(88403, 51455), tuple(73795, 47066), tuple(26448, 82852), tuple(22829, 2894), tuple(30041, 92548), tuple(27733, 20608), tuple(70180, 19892), tuple(51650, 63440), tuple(76328, 13666), tuple(40514, 6677), tuple(2786, 51059), tuple(40809, 16499), tuple(10857, 82541), tuple(78221, 61067), tuple(17982, 51969), tuple(85369, 66965), tuple(47153, 47149), tuple(43965, 75796), tuple(82725, 60767), tuple(42407, 97249), tuple(51475, 81224), tuple(60957, 89414), tuple(33065, 21663), tuple(36601, 5290), tuple(95842, 67301), tuple(64630, 60398), tuple(55212, 35638), tuple(41750, 44235), tuple(75260, 82400), tuple(91291, 25843), tuple(6477, 8311), tuple(14919, 52306), tuple(66220, 33180), tuple(45736, 2313), tuple(37450, 64444), tuple(98614, 61344), tuple(75007, 50946), tuple(56701, 28117), tuple(66632, 5174), tuple(92323, 76613), tuple(6796, 73695), tuple(33696, 76280), tuple(86876, 5614), tuple(50863, 67993), tuple(36068, 17049), tuple(91912, 34271), tuple(70706, 1904), tuple(97798, 41117), tuple(68154, 72483), tuple(83862, 25578), tuple(61643, 17204), tuple(69974, 64232), tuple(77926, 19637), + SELECT (rand(), rand()) IN tuple(tuple(17258, 93148), tuple(4508, 52749), tuple(68660, 70017), tuple(77797, 23528), tuple(1136, 37393), tuple(53237, 15379), tuple(68370, 73211), tuple(15782, 54962), tuple(59432, 45415), tuple(68396, 920), tuple(96154, 21016), tuple(12700, 26887), tuple(88016, 43191), tuple(68153, 51575), tuple(91315, 40005), tuple(18070, 73178), tuple(86, 631), tuple(77717, 20324), tuple(3227, 76188), tuple(74960, 43147), tuple(77538, 19628), tuple(82292, 6525), tuple(24293, 12566), tuple(85244, 96287), tuple(93982, 1329), tuple(38064, 54723), tuple(83999, 45810), tuple(71921, 53673), tuple(88638, 9669), tuple(1959, 39535), tuple(82235, 95796), tuple(27907, 90975), tuple(42383, 91015), tuple(9948, 91514), tuple(81712, 47309), tuple(400, 25808), tuple(31791, 46948), tuple(39740, 36098), tuple(25943, 84598), tuple(99598, 52939), tuple(77134, 15845), tuple(40313, 72174), tuple(85017, 94036), tuple(36595, 14303), tuple(83961, 68078), tuple(55792, 72759), tuple(73574, 43606), tuple(9853, 63560), tuple(28580, 56721), tuple(74804, 41025), tuple(32095, 55657), tuple(52881, 63416), tuple(91368, 90310), tuple(23922, 38883), tuple(30592, 10758), tuple(66448, 61183), tuple(31880, 96697), tuple(11362, 20633), tuple(75331, 2015), tuple(71129, 8785), tuple(1115, 70955), tuple(7886, 83698), tuple(18961, 84556), tuple(16677, 43028), tuple(37347, 70220), tuple(31699, 71244), tuple(10578, 96159), tuple(67600, 39041), tuple(78791, 86687), tuple(21545, 54174), tuple(68774, 37637), tuple(46132, 81768), tuple(98413, 20605), tuple(2960, 23665), tuple(31507, 35719), tuple(96209, 18368), tuple(60558, 38035), tuple(21952, 3264), tuple(11834, 86458), tuple(21651, 17650), tuple(86276, 36087), tuple(18818, 24849), tuple(61951, 3390), tuple(59637, 62545), tuple(30346, 72253), tuple(36281, 2992), tuple(78340, 49872), tuple(94326, 93723), tuple(3416, 94405), tuple(12272, 8741), tuple(22600, 22095), tuple(57636, 37106), tuple(38702, 14889), tuple(70238, 11276), tuple(17325, 60648), tuple(16492, 41271), tuple(52100, 1304), tuple(93416, 7795), tuple(57209, 71008), tuple(48010, 36078), tuple(20384, 74420), tuple(77440, 34439), tuple(69224, 45099), tuple(30374, 33884), tuple(49038, 90140), tuple(1154, 84725), tuple(64926, 86985), tuple(91746, 73472), tuple(59757, 75755), tuple(45860, 71557), tuple(45833, 36526), tuple(74618, 73598), tuple(91360, 65168), tuple(58029, 30793), tuple(56332, 14973), tuple(99943, 96877), tuple(97454, 6450), tuple(64502, 77301), tuple(73182, 31853), tuple(76809, 83964), tuple(82916, 86188), tuple(78736, 65427), tuple(36495, 7422), tuple(76196, 2804), tuple(96117, 61093), tuple(9177, 26099), tuple(52942, 63007), tuple(48578, 47876), tuple(50638, 89903), tuple(7113, 97316), tuple(35301, 12750), tuple(47807, 7254), tuple(38217, 55418), tuple(56970, 41687), tuple(20527, 62886), tuple(358, 14021), tuple(64018, 18582), tuple(91740, 21683), tuple(81967, 53589), tuple(45437, 38450), tuple(45476, 67752), tuple(76851, 72072), tuple(7304, 60091), tuple(40097, 12897), tuple(39906, 29247), tuple(84262, 58734), tuple(30857, 43791), tuple(56087, 78929), tuple(20498, 45954), tuple(48726, 500), tuple(62723, 43763), tuple(28368, 30756), tuple(74048, 52403), tuple(15045, 95926), tuple(75542, 55384), tuple(52543, 22525), tuple(56001, 6935), tuple(11431, 46745), tuple(77731, 7310), tuple(36718, 59909), tuple(32235, 91254), tuple(92417, 25917), tuple(21782, 79277), tuple(46378, 87536), tuple(35324, 26075), tuple(6310, 76915), tuple(1551, 69473), tuple(50642, 68865), tuple(55190, 72934), tuple(49780, 21873), tuple(99466, 29686), tuple(90761, 13179), tuple(72959, 57033), tuple(20020, 90200), tuple(46186, 79105), tuple(73871, 52382), tuple(59559, 38801), tuple(59916, 16082), tuple(33610, 94966), tuple(46001, 45225), tuple(86679, 26469), tuple(77245, 91929), tuple(32887, 36623), tuple(11179, 46898), tuple(87881, 68087), tuple(45438, 47991), tuple(24950, 94525), tuple(91664, 51656), tuple(43914, 47805), tuple(15736, 96156), tuple(56346, 20283), tuple(85053, 48931), tuple(17790, 26179), tuple(96195, 55728), tuple(43765, 54807), tuple(44988, 89269), tuple(55911, 99411), tuple(52446, 47397), tuple(28346, 65442), tuple(96669, 68226), tuple(66194, 26848), tuple(37276, 55864), tuple(14116, 41583), tuple(18058, 16317), tuple(93136, 85318), tuple(35616, 86252), tuple(29222, 29969), tuple(33386, 85372), tuple(71094, 44238), tuple(27733, 31838), tuple(64626, 16692), tuple(52904, 97899), tuple(97619, 12663), tuple(50165, 4688), tuple(67557, 44053), tuple(69184, 66269), tuple(73164, 89705), tuple(39822, 15169), tuple(65499, 72808), tuple(30068, 63697), tuple(30154, 64235), tuple(97016, 58716), tuple(94366, 36592), tuple(1592, 16261), tuple(87985, 52102), tuple(12554, 23652), tuple(15909, 25292), tuple(2527, 91531), tuple(92139, 36031), tuple(28986, 30032), tuple(3038, 56314), tuple(32239, 26707), tuple(15973, 34901), tuple(70246, 39680), tuple(82529, 38132), tuple(45827, 74783), tuple(53665, 64111), tuple(55218, 84170), tuple(20466, 16130), tuple(55734, 71203), tuple(31438, 96906), tuple(66338, 85858), tuple(35988, 68511), tuple(78391, 15191), tuple(80747, 59213), tuple(5357, 11546), tuple(16822, 16607), tuple(36607, 41106), tuple(74949, 30739), tuple(45726, 64887), tuple(1524, 54847), tuple(37371, 89195), tuple(28726, 27788), tuple(22600, 44777), tuple(53999, 63625), tuple(84304, 98338), tuple(49260, 76480), tuple(74564, 53907), tuple(89867, 97096), tuple(60157, 61299), tuple(17165, 10146), tuple(56334, 36268), tuple(62114, 49222), tuple(22715, 23620), tuple(42830, 11539), tuple(41091, 69151), tuple(75471, 68364), tuple(18681, 43249), tuple(42738, 63219), tuple(35474, 98454), tuple(76815, 46024), tuple(66310, 36521), tuple(86095, 77013), tuple(63693, 77319), tuple(80731, 63031), tuple(95478, 92387), tuple(23787, 63724), tuple(46299, 68994), tuple(4800, 2460), tuple(9663, 80639), tuple(77231, 85814), tuple(81615, 11311), tuple(35638, 27340), tuple(13598, 14322), tuple(30657, 17238), tuple(90957, 96846), tuple(69962, 52140), tuple(41681, 65962), tuple(96836, 58177), tuple(36190, 11623), tuple(4231, 40500), tuple(43049, 41949), tuple(71177, 98492), tuple(30193, 39750), tuple(19744, 33204), tuple(63358, 30210), tuple(45638, 58918), tuple(43641, 38741), tuple(35598, 40932), tuple(33238, 36236), tuple(50835, 20968), tuple(25099, 34071), tuple(84986, 88456), tuple(35333, 1529), tuple(79771, 23985), tuple(647, 61658), tuple(9424, 11743), tuple(77766, 31528), tuple(77811, 86973), tuple(76403, 74377), tuple(55568, 79251), tuple(68858, 20762), tuple(68520, 66773), tuple(93598, 89823), tuple(8080, 82539), tuple(87760, 52247), tuple(25191, 16905), tuple(17837, 8339), tuple(85177, 59050), tuple(51680, 77374), tuple(3287, 43018), tuple(43479, 62141), tuple(34909, 46322), tuple(11869, 5885), tuple(96193, 58417), tuple(101, 47460), tuple(34937, 88582), tuple(83216, 88388), tuple(28571, 15292), tuple(66683, 62613), tuple(34478, 8924), tuple(2680, 89973), tuple(62438, 44460), tuple(11724, 4791), tuple(5383, 72888), tuple(88206, 67586), tuple(8124, 21690), tuple(28779, 75789), tuple(66791, 4757), tuple(6176, 47760), tuple(6403, 78084), tuple(78122, 35446), tuple(99494, 73608), tuple(39691, 89098), tuple(59182, 19484), tuple(25389, 98963), tuple(96487, 3692), tuple(76222, 67381), tuple(21199, 50358), tuple(95998, 58137), tuple(28777, 43913), tuple(14176, 60117), tuple(52257, 81703), tuple(14604, 13438), tuple(71301, 14401), tuple(19758, 66914), tuple(15506, 29873), tuple(87205, 29449), tuple(93295, 15930), tuple(63651, 11287), tuple(19785, 15966), tuple(30795, 75112), tuple(69462, 37655), tuple(18793, 85764), tuple(36240, 31236), tuple(98153, 73724), tuple(72491, 4223), tuple(66930, 35048), tuple(25686, 13269), tuple(13940, 13259), tuple(69163, 11235), tuple(1183, 86961), tuple(54323, 67315), tuple(85044, 60872), tuple(48875, 3683), tuple(43052, 92861), tuple(87574, 32969), tuple(92552, 80564), tuple(94832, 47682), tuple(72011, 80994), tuple(60182, 917), tuple(97788, 34169), tuple(66432, 47940), tuple(87468, 80954), tuple(35385, 68758), tuple(50555, 63710), tuple(55311, 44337), tuple(87065, 26514), tuple(84581, 98736), tuple(23212, 56499), tuple(75120, 72447), tuple(56087, 38285), tuple(58171, 45629), tuple(28401, 44319), tuple(70432, 27883), tuple(18891, 14646), tuple(26206, 49924), tuple(79957, 44914), tuple(56064, 27529), tuple(99090, 29197), tuple(49435, 340), tuple(53525, 65601), tuple(76998, 88349), tuple(50416, 70860), tuple(42506, 75290), tuple(34024, 13295), tuple(86663, 46523), tuple(88814, 231), tuple(57809, 21), tuple(84914, 84771), tuple(43042, 66892), tuple(17288, 33908), tuple(4934, 63195), tuple(50590, 1516), tuple(97843, 80208), tuple(20091, 86717), tuple(71566, 15929), tuple(19531, 23634), tuple(41646, 45549), tuple(89226, 82902), tuple(96683, 63386), tuple(31072, 53788), tuple(51135, 41099), tuple(78912, 65609), tuple(36094, 23603), tuple(88403, 51455), tuple(73795, 47066), tuple(26448, 82852), tuple(22829, 2894), tuple(30041, 92548), tuple(27733, 20608), tuple(70180, 19892), tuple(51650, 63440), tuple(76328, 13666), tuple(40514, 6677), tuple(2786, 51059), tuple(40809, 16499), tuple(10857, 82541), tuple(78221, 61067), tuple(17982, 51969), tuple(85369, 66965), tuple(47153, 47149), tuple(43965, 75796), tuple(82725, 60767), tuple(42407, 97249), tuple(51475, 81224), tuple(60957, 89414), tuple(33065, 21663), tuple(36601, 5290), tuple(95842, 67301), tuple(64630, 60398), tuple(55212, 35638), tuple(41750, 44235), tuple(75260, 82400), tuple(91291, 25843), tuple(6477, 8311), tuple(14919, 52306), tuple(66220, 33180), tuple(45736, 2313), tuple(37450, 64444), tuple(98614, 61344), tuple(75007, 50946), tuple(56701, 28117), tuple(66632, 5174), tuple(92323, 76613), tuple(6796, 73695), tuple(33696, 76280), tuple(86876, 5614), tuple(50863, 67993), tuple(36068, 17049), tuple(91912, 34271), tuple(70706, 1904), tuple(97798, 41117), tuple(68154, 72483), tuple(83862, 25578), tuple(61643, 17204), tuple(69974, 64232), tuple(77926, 19637), tuple(64901, 88988), tuple(71424, 91703), tuple(91655, 17147), tuple(46872, 56530), tuple(44189, 98087), tuple(95939, 54420), tuple(72651, 68785), tuple(67624, 84875), tuple(92587, 87663), tuple(65275, 81256), tuple(53798, 2506), tuple(14702, 3638), tuple(71291, 50452), tuple(14909, 13903), tuple(66965, 26606), tuple(14127, 60345), tuple(35306, 1738), tuple(77234, 10468), tuple(53521, 41218), tuple(80681, 82583), tuple(44227, 26521), tuple(32263, 21482), tuple(82270, 56963), tuple(50580, 80567), tuple(11593, 22346), tuple(20074, 26867), tuple(73126, 28667), tuple(62996, 24317), tuple(20295, 57163), tuple(1506, 57668), tuple(69567, 45236), tuple(43366, 26001), tuple(88052, 40181), tuple(1599, 89349), tuple(36789, 1579), tuple(39895, 46673), tuple(30381, 3206), tuple(31723, 5625), tuple(19252, 31317), tuple(16932, 77149), tuple(48794, 34409), tuple(55986, 30328), tuple(47551, 75088), tuple(57363, 78365), tuple(95221, 63385), tuple(26449, 5733), tuple(96588, 53077), tuple(52980, 41140), tuple(8187, 85947), tuple(36723, 26520), tuple(23579, 38909), tuple(33350, 19275), tuple(63930, 19357), tuple(43536, 59941), tuple(31117, 77322), tuple(44638, 94812), tuple(44730, 99097), tuple(95108, 48170), tuple(57813, 49503), tuple(79959, 89436), tuple(86980, 62031), tuple(8275, 44009), tuple(36666, 94645), tuple(22064, 38882), tuple(40471, 16939), tuple(31156, 11337), tuple(13101, 96977), tuple(17906, 26835), tuple(89861, 51405), tuple(73369, 67946), tuple(99141, 58572), tuple(27131, 98703), tuple(15900, 43412), tuple(51768, 93125), tuple(78579, 46689), tuple(23029, 13895), tuple(60870, 55830), tuple(22553, 8236), tuple(76449, 96207), tuple(83766, 51024), tuple(27630, 50614), tuple(53484, 90104), tuple(77626, 21944), tuple(46755, 41583), tuple(53616, 34240), tuple(94159, 44415), tuple(13914, 90059), tuple(44387, 89012), tuple(27499, 64579), tuple(83415, 30809), tuple(77558, 82619), tuple(88880, 9814), tuple(8466, 4424), tuple(43598, 91921), tuple(24695, 3349), tuple(46295, 65208), tuple(51256, 82461), tuple(49126, 93012), tuple(16186, 96585), tuple(43284, 22655), tuple(93130, 90393), tuple(77495, 34372), tuple(85509, 65856), tuple(86662, 61906), tuple(50988, 44393), tuple(29828, 17737), tuple(91651, 35308), tuple(29796, 49716), tuple(14019, 87751), tuple(29688, 71207), tuple(82845, 19100), tuple(11989, 50132), tuple(21158, 99905), tuple(54732, 42547), tuple(32314, 12851), tuple(46405, 43794), tuple(87849, 45643), tuple(53524, 21212), tuple(61925, 75491), tuple(12498, 21937), tuple(30185, 69475), tuple(48421, 52487), tuple(15112, 90935), tuple(33187, 17801), tuple(61704, 25514), tuple(17889, 23917), tuple(18758, 57197), tuple(7693, 47232), tuple(47905, 24618), tuple(11494, 78950), tuple(95662, 54561), tuple(8075, 33909), tuple(90427, 46065), tuple(73962, 19821), tuple(50691, 79400), tuple(58218, 4881), tuple(94106, 2509), tuple(60633, 55169), tuple(49600, 83054), tuple(23339, 13270), tuple(70262, 58946), tuple(48417, 97266), tuple(27629, 46905), tuple(74465, 75514), tuple(41687, 2564), tuple(12814, 19492), tuple(78899, 30168), tuple(17745, 35206), tuple(37972, 35296), tuple(22288, 80001), tuple(68026, 36558), tuple(40187, 12234), tuple(92380, 22866), tuple(56488, 64402), tuple(41404, 62562), tuple(47802, 45287), tuple(83302, 85215), tuple(58999, 85776), tuple(35158, 16804), tuple(13416, 94146), tuple(62953, 28243), tuple(83290, 19103), tuple(4564, 21789), tuple(64468, 20927), tuple(25582, 47206), tuple(57810, 18693), tuple(28938, 97986), tuple(61704, 14838), tuple(19214, 3232), tuple(12911, 25438), tuple(85802, 28837), tuple(56506, 89458), tuple(66392, 47773), tuple(68190, 43841), tuple(43044, 52214), tuple(57886, 32830), tuple(15943, 59771), tuple(37081, 89294), tuple(4032, 32960), tuple(46931, 85790), tuple(69656, 72737), tuple(28217, 39872), tuple(86170, 42776), tuple(55116, 51495), tuple(90485, 45274), tuple(60773, 36788), tuple(2193, 2636), tuple(70222, 62086), tuple(75720, 70712), tuple(17549, 51460), tuple(23609, 31515), tuple(70254, 39825), tuple(63762, 11061), tuple(13107, 15394), tuple(45916, 72130), tuple(91558, 86662), tuple(99524, 69106), tuple(93073, 29881), tuple(31724, 3007), tuple(69051, 59452), tuple(59701, 86760), tuple(4967, 82028), tuple(57404, 48226), tuple(71829, 79910), tuple(23714, 62439), tuple(73881, 67618), tuple(63269, 40085), tuple(6164, 23415), tuple(48156, 93907), tuple(18627, 16570), tuple(6676, 22991), tuple(36916, 41488), tuple(99079, 13264), tuple(32533, 99243), tuple(55505, 63339), tuple(89564, 3290), tuple(24886, 34916), tuple(91310, 9343), tuple(49779, 12740), tuple(26320, 3406), tuple(57661, 5702), tuple(10765, 57881), tuple(5518, 47638), tuple(93148, 27438), tuple(73451, 24477), tuple(84075, 96822), tuple(58883, 58883), tuple(96812, 82388), tuple(30659, 59654), tuple(24498, 95808), tuple(25591, 21834), tuple(13090, 87704), tuple(76495, 17249), tuple(75975, 84318), tuple(55459, 70426), tuple(84256, 88604), tuple(79438, 43104), tuple(45331, 7495), tuple(63619, 11123), tuple(24772, 2601), tuple(63343, 14138), tuple(39957, 98339), tuple(55595, 17823), tuple(97676, 53933), tuple(91867, 25023), tuple(64677, 67859), tuple(43737, 34315), tuple(24800, 53968), tuple(93157, 17507), tuple(24264, 35273), tuple(33889, 507), tuple(10207, 40542), tuple(40213, 57800), tuple(38321, 74160), tuple(42391, 7651), tuple(80267, 94736), tuple(52473, 79634), tuple(17075, 2531), tuple(8595, 75890), tuple(31496, 50367), tuple(16069, 79896), tuple(70067, 200), tuple(23420, 49517), tuple(1628, 45646), tuple(8916, 36794), tuple(72294, 88976), tuple(40603, 86008), tuple(91871, 71098), tuple(5447, 70998), tuple(24152, 17561), tuple(65046, 34951), tuple(56950, 9292), tuple(19244, 31385), tuple(74693, 31813), tuple(97343, 21572), tuple(38834, 135), tuple(79717, 62486), tuple(38, 10308), tuple(58035, 71344), tuple(85802, 81079), tuple(5943, 156), tuple(38735, 38867), tuple(3803, 99366), tuple(15853, 19408), tuple(62988, 62008), tuple(8316, 44684), tuple(17035, 71012), tuple(48584, 2117), tuple(75425, 37336), tuple(2405, 50420), tuple(43653, 28836), tuple(12394, 69430), tuple(54522, 4954), tuple(33359, 148), tuple(41018, 82851), tuple(79995, 55417), tuple(65008, 32342), tuple(36547, 88185), tuple(8131, 7054), tuple(38980, 20146), tuple(27976, 63039), tuple(53119, 67009), tuple(40043, 98393), tuple(29333, 51980), tuple(85818, 98405), tuple(77956, 20099), tuple(99747, 16916), tuple(11597, 50181), tuple(40961, 8262), tuple(75103, 13912), tuple(62339, 69155), tuple(3869, 85481), tuple(7053, 30956), tuple(33563, 53272), tuple(96178, 81751), tuple(99365, 88728), tuple(34447, 11164), tuple(62856, 30939), tuple(92486, 3357), tuple(56605, 35330), tuple(42180, 15137), tuple(83946, 62984), tuple(61869, 55711), tuple(52880, 49871), tuple(44588, 27387), tuple(16332, 24496), tuple(1781, 13508), tuple(56674, 95773), tuple(21328, 19628), tuple(96455, 24155), tuple(14302, 74435), tuple(54053, 24590), tuple(86642, 22177), tuple(24089, 16186), tuple(70281, 4601), tuple(18552, 70708), tuple(95442, 5895), tuple(96714, 6293), tuple(43803, 45857), tuple(93257, 18497), tuple(90032, 85086), tuple(40566, 87233), tuple(32674, 73822), tuple(95599, 49334), tuple(62745, 51898), tuple(8245, 93882), tuple(14093, 40977), tuple(47215, 53001), tuple(59737, 68452), tuple(90937, 25354), tuple(43805, 82571), tuple(81953, 68572), tuple(37298, 96262), tuple(94899, 65066), tuple(34772, 80762), tuple(55469, 1186), tuple(8734, 91665), tuple(18622, 51150), tuple(85200, 39575), tuple(65381, 15979), tuple(89734, 89656), tuple(64712, 53691), tuple(87187, 58256), tuple(8476, 89694), tuple(49935, 35239), tuple(63730, 34982), tuple(27687, 91571), tuple(87543, 15350), tuple(85208, 18781), tuple(14783, 2574), tuple(44699, 666), tuple(56440, 87617), tuple(32732, 49301), tuple(76725, 3895), tuple(10419, 90580), tuple(34725, 69476), tuple(14831, 81588), tuple(93924, 38057), tuple(38528, 99060), tuple(57136, 44206), tuple(74685, 99559), tuple(43083, 87511), tuple(43105, 35474), tuple(35582, 17560), tuple(5578, 98727), tuple(78947, 53865), tuple(32013, 95029), tuple(61552, 42674), tuple(52191, 49975), tuple(71566, 16403), tuple(78534, 16350), tuple(18520, 80501), tuple(29114, 46547), tuple(11488, 5069), tuple(89591, 82384), tuple(13741, 42318), tuple(74385, 58849), tuple(49739, 63421), tuple(83821, 6676), tuple(51997, 93321), tuple(36677, 81768), tuple(37915, 73495), tuple(47175, 6086), tuple(39989, 83110), tuple(6489, 48112), tuple(88822, 20370), tuple(12846, 13952), tuple(28930, 20879), tuple(25139, 84552), tuple(76434, 2665), tuple(55145, 31523), tuple(21177, 18630), tuple(81077, 96275), tuple(61006, 30845), tuple(77722, 62651), tuple(61181, 72545), tuple(93838, 84287), tuple(59300, 19014), tuple(75076, 97980), tuple(76979, 1473), tuple(48409, 13097), tuple(51718, 5325), tuple(36522, 72119), tuple(60917, 18995), tuple(61469, 42853), tuple(34387, 37322), tuple(38684, 28120), tuple(64136, 8559), tuple(15368, 99424), tuple(97824, 7864), tuple(33833, 72029), tuple(7024, 9961), tuple(49400, 66220), tuple(63025, 97179), tuple(6135, 98878), tuple(19873, 8438), tuple(3963, 35670), tuple(65186, 89423), tuple(26653, 65943), tuple(83132, 67000), tuple(82578, 35007), tuple(42680, 60479), tuple(71102, 98589), tuple(74842, 94010), tuple(22931, 33725), tuple(46537, 42629), tuple(75793, 48115), tuple(21630, 92454), tuple(97993, 81332), tuple(25747, 31814), tuple(91231, 65953), tuple(91981, 12219), tuple(64719, 16254), tuple(60914, 8334), tuple(15887, 96432), tuple(42110, 28837), tuple(7295, 83147), tuple(50334, 7053), tuple(3949, 33594), tuple(1524, 98230), tuple(17265, 98024), tuple(75969, 36232), tuple(89538, 5212), tuple(13444, 55946), tuple(69823, 81848), tuple(32578, 74024), tuple(52018, 98290), tuple(59118, 40186), tuple(61002, 16977), tuple(69537, 44780), tuple(92, 13937), tuple(33715, 42663), tuple(46347, 8312), tuple(86196, 59301), tuple(17128, 85014), tuple(26429, 57682), tuple(45888, 99588), tuple(22750, 96110), tuple(46809, 49251), tuple(24521, 40071), tuple(287, 22115), tuple(11741, 36315), tuple(22742, 17581), tuple(35808, 3110), tuple(98904, 30407), tuple(4584, 13383), tuple(28585, 69669), tuple(94823, 29715), tuple(9551, 36389), tuple(77997, 45746), tuple(49894, 55722), tuple(23415, 69459), tuple(58246, 85144), tuple(74136, 18102), tuple(97366, 85724), tuple(34271, 51601), tuple(47535, 70883), tuple(59443, 90103), tuple(45213, 45811), tuple(62741, 86898), tuple(17324, 50034), tuple(62080, 25193), tuple(89524, 4421), tuple(13476, 51456), tuple(69198, 56718), tuple(58024, 22969), tuple(65210, 67941), tuple(32561, 44881), tuple(62295, 67448), tuple(66135, 95453), tuple(9417, 20443), tuple(82486, 23745), tuple(19185, 99041), tuple(40662, 91714), tuple(3423, 58624), tuple(4512, 74502), tuple(67772, 98023), tuple(69575, 75779), tuple(69107, 62805), tuple(517, 33801), tuple(47406, 7581), tuple(81108, 10546), tuple(12976, 47001), tuple(16742, 83811), tuple(44593, 82124), tuple(52731, 34642), tuple(81725, 20555), tuple(94126, 91919), tuple(24800, 59302), tuple(97253, 39249), tuple(71692, 10769), tuple(88721, 56321), tuple(7019, 69771), tuple(31464, 61774), tuple(29597, 19263), tuple(65557, 31875), tuple(28653, 69636), tuple(58074, 76848), tuple(15906, 80620), tuple(18259, 40193), tuple(99991, 4769), tuple(98935, 99269), tuple(12123, 60124), tuple(20787, 47346), tuple(13526, 33592), tuple(95370, 40350), tuple(17479, 42884), tuple(58368, 83218), tuple(63290, 74406), tuple(97030, 35102), tuple(45298, 27660), tuple(64593, 21262), tuple(76268, 82641), tuple(1107, 44044), tuple(21427, 79959), tuple(85180, 62412), tuple(7359, 1318), tuple(83618, 9762), tuple(1425, 55804), tuple(32874, 97943), tuple(68191, 38742), tuple(41715, 17902), tuple(3771, 15032), tuple(7848, 74950), tuple(33881, 40904), tuple(75295, 26151), tuple(75775, 13760), tuple(90262, 89822), tuple(88169, 18679), tuple(57506, 32356), tuple(94983, 44281), tuple(37385, 37432), tuple(18248, 48162), tuple(45573, 66278), tuple(25277, 72788), tuple(26977, 36778), tuple(26254, 61758), tuple(12860, 48026), tuple(96819, 3339), tuple(13134, 1173), tuple(26822, 53374), tuple(15989, 29698), tuple(11258, 54515), tuple(37866, 34928), tuple(22996, 26577), tuple(39952, 42732), tuple(6754, 70595), tuple(86245, 44669), tuple(47044, 34170), tuple(6789, 45220), tuple(31706, 2090), tuple(42582, 40023), tuple(35147, 46591), tuple(88210, 11307), tuple(53644, 7680), tuple(11280, 91075), tuple(42961, 65122), tuple(40066, 52185), tuple(20050, 6154), tuple(98440, 20393), tuple(88992, 75432), tuple(32386, 66731), tuple(36952, 34149), tuple(18453, 32715), tuple(84413, 10378), tuple(59440, 2374), tuple(45354, 85009), tuple(50382, 66510), tuple(64428, 95401), tuple(9336, 41760), tuple(26317, 91416), tuple(81941, 99504), tuple(26600, 53522), tuple(81069, 40236), tuple(51126, 27911), tuple(97144, 14243), tuple(62738, 50287), tuple(37372, 28962), tuple(12053, 9090), tuple(69492, 95524), tuple(68141, 52931), tuple(17276, 16487), tuple(69227, 25949), tuple(14143, 70193), tuple(7077, 53032), tuple(65463, 74082), tuple(94997, 66496), tuple(80443, 55832), tuple(66796, 5970), tuple(15852, 95662), tuple(81559, 97272), tuple(55851, 18977), tuple(91142, 48976), tuple(91143, 950), tuple(79225, 31004), tuple(61310, 20760), tuple(74541, 90842), tuple(80322, 11630), tuple(84631, 544), tuple(66785, 86591), tuple(25650, 63252), tuple(59635, 18586), tuple(2964, 6741), tuple(37091, 71148), tuple(11984, 43077), tuple(87505, 62049), tuple(61925, 92290), tuple(18808, 3937), tuple(8300, 33268), tuple(70850, 50661), tuple(86024, 73730), tuple(85161, 47116), tuple(50193, 89155), tuple(37773, 40845), tuple(9251, 41688), tuple(6940, 65399), tuple(42479, 95630), tuple(19401, 43102), tuple(48069, 36040), tuple(62760, 95013), tuple(394, 2641), tuple(32567, 29306), tuple(13870, 58835), tuple(98248, 47291), tuple(49803, 4523), tuple(40222, 12883), tuple(53576, 73105), tuple(88265, 23629), tuple(67865, 67875), tuple(33473, 27144), tuple(80219, 53893), tuple(74878, 47341), tuple(78070, 84803), tuple(30003, 5600), tuple(41103, 6145), tuple(83490, 81076), tuple(55059, 66736), tuple(45015, 10239), tuple(79555, 85819), tuple(81808, 34970), tuple(19235, 85480), tuple(91807, 52177), tuple(40887, 87009), tuple(5003, 2687), tuple(64964, 88122), tuple(765, 94893), tuple(93573, 20504), tuple(28854, 38438), tuple(94244, 93475), tuple(72996, 84801), tuple(75427, 81692), tuple(63161, 98637), tuple(18814, 61343), tuple(22863, 60110), tuple(8949, 12694), tuple(19675, 94313), tuple(43857, 74073), tuple(15737, 58218), tuple(48895, 68474), tuple(22220, 92926), tuple(69055, 50282), tuple(40532, 74934), tuple(59062, 66405), tuple(85784, 87704), tuple(58494, 88222), tuple(2260, 20401), tuple(73112, 99666), tuple(46739, 95433), tuple(21179, 85119), tuple(11545, 38801), tuple(59993, 50866), tuple(10086, 4709), tuple(70560, 29611), tuple(27095, 89017), tuple(6896, 2279), tuple(92506, 5013), tuple(48600, 90491), tuple(18782, 54638), tuple(54337, 82734), tuple(52054, 13481), tuple(38297, 56559), tuple(15998, 30591), tuple(89789, 7522), tuple(18149, 28725), tuple(3532, 28625), tuple(70934, 49617), tuple(84599, 55664), tuple(74229, 52269), tuple(55431, 11893), tuple(32807, 72543), tuple(83882, 53025), tuple(11490, 83442), tuple(14844, 88612), tuple(12526, 45953), tuple(906, 2231), tuple(68240, 95612), tuple(18818, 31535), tuple(57774, 91290), tuple(67250, 67400), tuple(77332, 23550), tuple(42332, 57775), tuple(28792, 11539), tuple(19108, 34608), tuple(12399, 38591), tuple(7329, 10740), tuple(84288, 50928), tuple(29461, 17629), tuple(63884, 88489), tuple(47479, 61085), tuple(75357, 57255), tuple(60107, 94046), tuple(32934, 66312), tuple(28615, 42600), tuple(55553, 85213), tuple(57838, 91426), tuple(9783, 11513), tuple(73677, 28821), tuple(75408, 75561), tuple(22995, 59224), tuple(74874, 54145), tuple(18513, 75901), tuple(46440, 69414), tuple(36072, 22263), tuple(60560, 73325), tuple(69967, 93358), tuple(75949, 98634), tuple(3688, 57991), tuple(43482, 94541), tuple(40922, 31011), tuple(57763, 74497), tuple(93576, 96392), tuple(83038, 80656), tuple(47757, 87045), tuple(14061, 53465), tuple(65619, 33775), tuple(11341, 6702), tuple(6249, 87358), tuple(15766, 85937), tuple(13135, 93945), tuple(24495, 95900), tuple(80359, 1739), tuple(15468, 73426), tuple(49240, 44999), tuple(82839, 90808), tuple(87438, 75613), tuple(348, 73144), tuple(99523, 85853), tuple(21557, 70210), tuple(64933, 1672), tuple(38154, 17477), tuple(97136, 67363), tuple(96491, 8038), tuple(97981, 3434), tuple(54372, 27038), tuple(88480, 86675), tuple(21028, 21083), tuple(43197, 4440), tuple(31702, 78290), tuple(66631, 24438), tuple(11482, 17922), tuple(90351, 39503), tuple(46186, 32439), tuple(73828, 6640), tuple(56916, 26029), tuple(62840, 1815), tuple(20281, 28488), tuple(18211, 30043), tuple(65211, 93012), tuple(43614, 58012), tuple(90322, 77343), tuple(64293, 94525), tuple(59489, 39760), tuple(93219, 78440), tuple(74613, 9732), tuple(38085, 19191), tuple(58029, 48186), tuple(88762, 1764), tuple(28627, 21993), tuple(49975, 41225), tuple(70486, 43480), tuple(82764, 96425), tuple(27218, 78327), tuple(17844, 73333), tuple(70463, 37629), tuple(10500, 33826), tuple(97343, 66575), tuple(82833, 51210), tuple(77353, 45073), tuple(27163, 39728), tuple(78076, 46691), tuple(80302, 39342), tuple(77142, 1319), tuple(87403, 80110), tuple(53805, 27786), tuple(50558, 74264), tuple(83146, 31358), tuple(11567, 4438), tuple(30041, 54287), tuple(91731, 18496), tuple(57591, 93894), tuple(72534, 59009), tuple(98064, 59148), tuple(69626, 66615), tuple(20951, 43949), tuple(61960, 68060), tuple(48892, 67918), tuple(61321, 56222), tuple(75424, 77260), tuple(4916, 81929), tuple(68892, 81531), tuple(28096, 28548), tuple(62016, 107), tuple(8593, 12030), tuple(66743, 36772), tuple(60174, 15106), tuple(52844, 1923), tuple(34768, 22065), tuple(88988, 62910), tuple(79214, 2998), tuple(25675, 31376), tuple(69959, 3614), tuple(43885, 31708), tuple(12206, 46548), tuple(69924, 19343), tuple(12984, 38980), tuple(58250, 69438), tuple(2580, 48684), tuple(38112, 37124), tuple(21842, 43150), tuple(59384, 21921), tuple(19908, 46678), tuple(73396, 79529), tuple(8274, 1557), tuple(36975, 65519), tuple(81069, 18712), tuple(13692, 9148), tuple(60617, 84762), tuple(75749, 66154), tuple(80375, 24553), tuple(4257, 47056), tuple(76880, 7687), tuple(40714, 43448), tuple(79112, 74791), tuple(33119, 72730), tuple(17670, 89183), tuple(51614, 3921), tuple(21247, 39857), tuple(86756, 67673), tuple(32792, 70035), tuple(5917, 7197), tuple(1762, 23130), tuple(6455, 63664), tuple(32806, 3729), tuple(60469, 20511), tuple(12522, 15149), tuple(98106, 79338), tuple(84754, 11162), tuple(52058, 17973), tuple(28789, 1521), tuple(32766, 36325), tuple(78914, 40453), tuple(70297, 71854), tuple(9313, 45190), tuple(54559, 66227), tuple(22342, 43860), tuple(44152, 84294), tuple(36913, 93173), tuple(88523, 36338), tuple(82234, 71140), tuple(8328, 22947), tuple(73250, 88125), tuple(74356, 16820), tuple(94472, 37349), tuple(23126, 87806), tuple(40315, 88729), tuple(19935, 19145), tuple(93312, 65719), tuple(8477, 33108), tuple(86660, 69525), tuple(75557, 66964), tuple(60437, 57494), tuple(94419, 42524), tuple(95372, 72274), tuple(49866, 85685), tuple(96808, 39404), tuple(62961, 72507), tuple(38634, 70815), tuple(91379, 42430), tuple(66359, 98699), tuple(24382, 4186), tuple(4003, 77760), tuple(87840, 75265), tuple(57641, 68871), tuple(9773, 15942), tuple(5664, 51289), tuple(47923, 31308), tuple(58632, 82468), tuple(14097, 71829), tuple(1838, 97710), tuple(70433, 11364), tuple(82363, 97879), tuple(25257, 20615), tuple(18249, 6758), tuple(98581, 13639), tuple(3290, 72449), tuple(74546, 79380), tuple(97254, 44448), tuple(80316, 31760), tuple(40516, 94809), tuple(14444, 88981), tuple(9693, 10259), tuple(83795, 95485), tuple(70201, 81014), tuple(66644, 16761), tuple(35529, 82718), tuple(75774, 73476), tuple(80139, 3957), tuple(34803, 80689), tuple(92085, 46499), tuple(97871, 8004), tuple(67369, 11354), tuple(43578, 81596), tuple(94695, 44963), tuple(93741, 41629), tuple(16005, 96652), tuple(37918, 69012), tuple(89832, 56041), tuple(51798, 32386), tuple(89749, 27647), tuple(76279, 7990), tuple(31746, 1346), tuple(40841, 20480), tuple(99942, 24473), tuple(78495, 99194), tuple(13588, 57088), tuple(22183, 42297), tuple(82707, 34435), tuple(45026, 12747), tuple(8000, 93211), tuple(40453, 13025), tuple(44100, 39880), tuple(83900, 56474), tuple(87691, 42802), tuple(82000, 63867), tuple(76627, 84731), tuple(112, 92774), tuple(34749, 97737), tuple(59262, 57169), tuple(95571, 44144), tuple(74310, 68970), tuple(63232, 92744), tuple(53698, 21610), tuple(39969, 75475), tuple(39942, 28713), tuple(81230, 50140), tuple(97953, 96528), tuple(86144, 48041), tuple(96677, 49677), tuple(22051, 48183), tuple(33436, 54784), tuple(5553, 11920), tuple(67057, 17115), tuple(57736, 72309), tuple(8086, 85329), tuple(72623, 94949), tuple(13096, 48992), tuple(63153, 56337), tuple(30462, 1036), tuple(75000, 24048), tuple(62635, 50128), tuple(91480, 83131), tuple(25929, 79809), tuple(96237, 76974), tuple(59645, 20603), tuple(31850, 29754), tuple(91070, 36568), tuple(30191, 33785), tuple(86776, 67259), tuple(49073, 39179), tuple(16121, 73834), tuple(84217, 52951), tuple(95866, 47244), tuple(63326, 73460), tuple(134, 91953), tuple(48189, 86069), tuple(42971, 3700), tuple(28643, 10479), tuple(80151, 7446), tuple(78798, 2655), tuple(39135, 69364), tuple(80244, 24904), tuple(22818, 74964), tuple(26753, 82419), tuple(16858, 5212), tuple(79891, 11215), tuple(785, 46103), tuple(12559, 24617), tuple(73601, 71490), tuple(70342, 7099), tuple(73330, 6665), tuple(11903, 28194), tuple(16375, 37746), tuple(86132, 51788), tuple(90345, 68366), tuple(5464, 78338), tuple(23569, 83141), tuple(17904, 94046), tuple(35868, 60017), tuple(22591, 93373), tuple(70584, 72116), tuple(49331, 34312), tuple(16180, 91286), tuple(58494, 65441), tuple(9336, 52671), tuple(32523, 26734), tuple(40205, 83549), tuple(28810, 96876), tuple(44703, 38944), tuple(46981, 37157), tuple(8582, 7529), tuple(59718, 71700), tuple(62545, 73716), tuple(6531, 23200), tuple(30528, 59720), tuple(57152, 84660), tuple(16232, 67946), tuple(60446, 45983), tuple(68737, 54959), tuple(57795, 73107), tuple(26930, 35938), tuple(9844, 44760), tuple(3716, 79020), tuple(99126, 8264), tuple(66120, 16151), tuple(50616, 25765), tuple(93340, 95875), tuple(34103, 88003), tuple(14879, 99758), tuple(49188, 6087), tuple(89858, 42861), tuple(36730, 72076), tuple(25069, 26403), tuple(98183, 48108), tuple(3229, 5367), tuple(59306, 80078), tuple(61144, 58598), tuple(72600, 98765), tuple(57701, 23177), tuple(10176, 11553), tuple(82964, 13697), tuple(7788, 28538), tuple(39943, 97491), tuple(56261, 17781), tuple(2458, 1892), tuple(6679, 45554), tuple(42171, 66222), tuple(24420, 44115), tuple(35852, 41965), tuple(50196, 49555), tuple(34718, 60734), tuple(6932, 61638), tuple(69472, 56723), tuple(489, 97620), tuple(41335, 90578), tuple(1333, 92787), tuple(97883, 64754), tuple(14208, 22097), tuple(75776, 5938), tuple(67446, 61518), tuple(58743, 45162), tuple(34749, 81243), tuple(71451, 91991), tuple(27804, 41836), tuple(45274, 8039), tuple(17593, 24498), tuple(8801, 38559), tuple(87460, 7109), tuple(50075, 18284), tuple(84043, 82146), tuple(62932, 25018), tuple(89647, 56768), tuple(59920, 80801), tuple(56357, 35142), tuple(97376, 58181), tuple(70715, 91103), tuple(90829, 78985), tuple(29776, 13275), tuple(30546, 42320), tuple(99266, 35340), tuple(21234, 61062), tuple(39239, 10745), tuple(45990, 715), tuple(47047, 6619), tuple(4270, 94575), tuple(90009, 72203), tuple(25629, 2691), tuple(67926, 89112), tuple(46990, 61101), tuple(22355, 69536), tuple(1977, 56723), tuple(54681, 34041), tuple(83819, 7024), tuple(81235, 7093), tuple(16659, 87135), tuple(49384, 32135), tuple(42204, 17362), tuple(90585, 70374), tuple(51255, 1), tuple(31600, 70085), tuple(90189, 95778), tuple(57349, 87789), tuple(83384, 93771), tuple(20718, 15529), tuple(10644, 53591), tuple(84103, 62101), tuple(91340, 48382), tuple(82854, 84420), tuple(12561, 53517), tuple(64835, 45362), tuple(54154, 75841), tuple(46498, 31175), tuple(75035, 49552), tuple(9306, 53403), tuple(68851, 49139), tuple(13463, 42107), tuple(2322, 36695), tuple(55953, 12098), tuple(60656, 80482), tuple(78987, 19632), tuple(31228, 18523), tuple(98972, 80489), tuple(32367, 98405), tuple(25139, 5164), tuple(5692, 60610), tuple(36535, 70097), tuple(80542, 74320), tuple(87984, 46750), tuple(98201, 41341), tuple(35217, 46813), tuple(81795, 69057), tuple(83927, 41032), tuple(60149, 26087), tuple(39954, 48361), tuple(64485, 61448), tuple(87185, 14580), tuple(74559, 93251), tuple(88544, 83366), tuple(74015, 15864), tuple(78623, 69719), tuple(16941, 80710), tuple(16315, 58313), tuple(47277, 59107), tuple(16067, 66290), tuple(63906, 59891), tuple(20754, 67817), tuple(44428, 10652), tuple(95960, 99045), tuple(52163, 26221), tuple(65566, 22057), tuple(26836, 38898), tuple(57107, 79274), tuple(39020, 74857), tuple(53540, 84159), tuple(76646, 44324), tuple(27967, 40171), tuple(28710, 56332), tuple(84036, 28711), tuple(68742, 57241), tuple(40535, 34737), tuple(62681, 85386), tuple(30472, 58405), tuple(85086, 33013), tuple(67059, 47481), tuple(30441, 55098), tuple(97892, 71991), tuple(90296, 42905), tuple(22441, 18863), tuple(19606, 77242), tuple(11206, 58380), tuple(23901, 49962), tuple(84094, 33761), tuple(64400, 28093), tuple(64228, 94543), tuple(71874, 20871), tuple(25385, 73117), tuple(63398, 20999), tuple(77547, 51893), tuple(80783, 65858), tuple(39807, 80754), tuple(10336, 90318), tuple(7826, 55346), tuple(30206, 10711), tuple(94411, 67364), tuple(33509, 14329), tuple(65350, 17006), tuple(65999, 55699), tuple(82753, 61081), tuple(38851, 11896), tuple(15155, 48635), tuple(19985, 75204), tuple(37144, 5344), tuple(26173, 39587), tuple(61111, 30966), tuple(16180, 22987), tuple(60707, 43599), tuple(30136, 74118), tuple(7880, 43857), tuple(97445, 30233), tuple(62700, 24828), tuple(90914, 89452), tuple(64131, 56925), tuple(25259, 39132), tuple(47104, 43950), tuple(93891, 21913), tuple(84573, 91029), tuple(8604, 79858), tuple(33141, 25534), tuple(12468, 90413), tuple(97063, 76359), tuple(80826, 26061), tuple(64013, 99099), tuple(82158, 38882), tuple(25799, 7564), tuple(25477, 69847), tuple(73374, 58520), tuple(48230, 9453), tuple(91424, 72273), tuple(64893, 11750), tuple(46753, 48434), tuple(15974, 94633), tuple(14872, 27027), tuple(14527, 21313), tuple(25660, 64644), tuple(54196, 15138), tuple(6313, 10911), tuple(36168, 47170), tuple(45346, 76), tuple(10305, 60286), tuple(65283, 39977), tuple(21804, 37972), tuple(65389, 86954), tuple(90674, 64458), tuple(15838, 22392), tuple(43540, 42503), tuple(49584, 67828), tuple(56711, 87887), tuple(40075, 73696), tuple(23832, 91552), tuple(39002, 65562), tuple(20451, 64664), tuple(70783, 92171), tuple(29319, 57694), tuple(56217, 44247), tuple(52856, 57873), tuple(80560, 90902), tuple(31068, 11280), tuple(46996, 34739), tuple(57527, 4554), tuple(8410, 25816), tuple(12269, 38319), tuple(88054, 49939), tuple(337, 13231), tuple(56432, 68236), tuple(74841, 21476), tuple(96006, 15712), tuple(87145, 91660), tuple(58090, 55111), tuple(10310, 79789), tuple(5734, 79710), tuple(98992, 69026), tuple(77033, 5734), tuple(43338, 42635), tuple(23898, 28669), tuple(62708, 81652), tuple(41279, 51722), tuple(93444, 26355), tuple(62046, 52199), tuple(71492, 58736), tuple(7379, 62581), tuple(8592, 71885), tuple(75026, 40387), tuple(46696, 3939), tuple(9787, 88907), tuple(86356, 363), tuple(97479, 20358), tuple(77363, 65553), tuple(44036, 22178), tuple(98279, 64612), tuple(3615, 411), tuple(77003, 93018), tuple(41605, 88489), tuple(55992, 83614), tuple(19493, 21633), tuple(34639, 97064), tuple(94602, 89289), tuple(45853, 26299), tuple(84170, 73386), tuple(9221, 51439), tuple(41513, 68166), tuple(37170, 17690), tuple(82511, 59246), tuple(96674, 27574), tuple(99301, 45675), tuple(42716, 41520), tuple(56623, 49130), tuple(84100, 76804), tuple(73855, 97007), tuple(73303, 26912), tuple(37151, 23837), tuple(49190, 97104), tuple(23487, 45628), tuple(87763, 46550), tuple(65111, 92605), tuple(80481, 8151), tuple(83949, 18930), tuple(81749, 27244), tuple(37449, 3023), tuple(28303, 51545), tuple(96441, 93242), tuple(22082, 43254), tuple(35135, 68407), tuple(37712, 48709), tuple(5111, 26774), tuple(15532, 74246), tuple(93605, 83583), tuple(21491, 66472), tuple(38922, 53076), tuple(55455, 54432), tuple(955, 44063), tuple(311, 91630), tuple(53554, 4522), tuple(29927, 65668), tuple(7525, 16035), tuple(44093, 76745), tuple(21481, 78198), tuple(76875, 5306), tuple(56126, 76437), tuple(96534, 16880), tuple(85600, 68336), tuple(4479, 81002), tuple(80414, 11593), tuple(8186, 61147), tuple(5624, 32879), tuple(79312, 20995), tuple(40407, 41512), tuple(91261, 66022), tuple(93228, 75364), tuple(21136, 40111), tuple(92148, 60681), tuple(42549, 7944), tuple(60157, 15040), tuple(63562, 88365), tuple(69056, 72713), tuple(78263, 89223), tuple(3776, 33039), tuple(30042, 59984), tuple(64567, 20977), tuple(24720, 39157), tuple(63582, 75653), tuple(45363, 20249), tuple(58093, 53833), tuple(27918, 93306), tuple(25791, 92686), tuple(15904, 862), tuple(72093, 19257), tuple(64125, 88986), tuple(41717, 27989), tuple(43165, 98675), tuple(76840, 48170), tuple(64508, 3535), tuple(91964, 33435), tuple(96686, 88673), tuple(66648, 64594), tuple(17927, 30539), tuple(73615, 22800), tuple(18580, 48077), tuple(59803, 48202), From 3e5f85ddd3949eb1b61b6c21f795606b6c97a8ed Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 13 May 2021 12:20:02 +0300 Subject: [PATCH 04/40] run short queries longer --- docker/test/performance-comparison/perf.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 2588b9f4213..286a5a99b2f 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -356,10 +356,11 @@ for query_index in queries_to_run: # For very short queries we have a special mode where we run them for at # least some time. The recommended lower bound of run time for "normal" # queries is about 0.1 s, and we run them about 10 times, giving the - # time per query per server of about one second. Use this value as a - # reference for "short" queries. + # time per query per server of about one second. Run "short" queries + # for longer time, because they have a high percentage of overhead and + # might give less stable results. if is_short[query_index]: - if server_seconds >= 2 * len(this_query_connections): + if server_seconds >= 8 * len(this_query_connections): break # Also limit the number of runs, so that we don't go crazy processing # the results -- 'eqmed.sql' is really suboptimal. From 474eff5f9e451d4de9ad4c00340d7efc24074c0b Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Fri, 14 May 2021 13:14:52 +0300 Subject: [PATCH 05/40] fix --- tests/performance/joins_in_memory_pmj.xml | 2 +- tests/performance/set_index.xml | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/performance/joins_in_memory_pmj.xml b/tests/performance/joins_in_memory_pmj.xml index d122dba72c3..05d1876cb06 100644 --- a/tests/performance/joins_in_memory_pmj.xml +++ b/tests/performance/joins_in_memory_pmj.xml @@ -1,4 +1,4 @@ - + CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index 03c249c26ff..a198f604215 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -61,8 +61,9 @@ (81277, 26395), (57176, 69906), (55080, 20726), (65843, 9495), (27703, 24879), (60594, 53447), (72896, 95513), (87884, 50244), (25281, 16427), (14122, 6399), (65765, 45679), (38160, 38650), (76820, 91595), (74663, 37143), (35220, 16865), (54664, 88840), (7671, 19781), (7255, 87276), (92182, 47030), (98922, 49381), (21954, 20742), (54674, 19730), (74694, 81672), (31070, 97794), (22759, 3530), (6085, 72621), (17392, 26996), (82220, 41192), (48225, 41272), (42850, 54369), (10817, 25978), (18889, 81907), (33157, 51203), (64325, 5137), (54695, 50597), (32244, 99170), (64264, 23137), (22912, 47267), (70570, 51635), (96224, 20571), (91110, 65969), (2598, 66368), (80926, 52434), (1538, 77141), (10149, 21484), (4981, 58501), (5154, 55414), (66791, 35619), (9905, 90667), (83595, 75525), (31989, 37492), (41740, 48920), (33876, 10878), (48654, 50120), (85890, 46200), (67100, 30387), (76475, 38916), (1654, 1657), (93842, 71472), (14560, 97747), (24866, 74187), (11477, 2492), (94590, 95965), (29363, 58015), (58783, 5304), (27337, 87412), (17695, 41885), (91837, 75700), (271, 49400), (33438, 84607), (78520, 46982), (38817, 66995), (2031, 58147), (80354, 75561), (7752, 55141), (22715, 29219), (10069, 69521), (1711, 23325), (88218, 94290), (44895, 32844), (62152, 964), (32945, 57419), (19030, 51353), (53097, 36525), (76592, 2389), (53949, 27282), (10569, 90204), (94318, 48527), (13855, 27099), (41107, 11962), (39143, 81444), (66004, 38387), (55906, 61394), (34933, 33952), (53868, 82829), (86213, 80445), (40351, 56305), (97656, 90533), (48021, 7262), (49406, 38294), (57170, 7675), (7040, 8039), (65129, 76668), (7554, 74457), (65930, 2421), (12739, 37328), (17174, 68325), (63207, 21315), (70066, 44503), (19112, 8081), (80329, 65152), (76480, 39600), (75262, 45432), (24382, 67054), (75360, 90438), (21786, 19182), (2050, 82428), (34831, 92670), (69746, 44939), (6130, 58630), (62478, 14374), (35730, 71761), (20641, 66145), (86551, 64076), (37559, 79379), (8215, 82203), (33701, 9187), (22679, 38196), (17476, 25994), (4391, 62658), (87726, 32997), (22552, 65818), (18115, 42587), (64374, 97689), (64896, 49572), (98479, 75547), (881, 32287), (25305, 16000), (295, 67454), (67642, 87363), (50945, 31544), (95966, 9972), (44139, 28252), (83682, 88471), (34615, 665), (13190, 85527), (32195, 81130), (22310, 12209), (27910, 60272), (40133, 5905), (29352, 36686), (76443, 64096), (79522, 79616), (63465, 96631), (95604, 90216), (88548, 12411), (63732, 50843), (1478, 11113), (41472, 43329), (35207, 78855), (95093, 22581), (67812, 1348), (96472, 75947), (18984, 63538), (84782, 72178), (53524, 46634), (95525, 15476), (82732, 89272), (59603, 2032), (74431, 78243), (47712, 51835), (82671, 67076), (63302, 96469), (77655, 95524), (11099, 31832), (77967, 32724), (85070, 36190), (93260, 32825), (51348, 34162), (40912, 26841), (27227, 21822), (15979, 96933), (76990, 90541), (36121, 90437), (85571, 9605), (85987, 46881), (86738, 82375), (80188, 48238), (64608, 41916), (79866, 9442), (28100, 95607), (10136, 19628), (6078, 59019), (8971, 11986), (75703, 209), (81982, 98701), (30455, 66367), (92825, 84403), (73144, 4923), (62835, 75533), (90925, 56381), (83853, 86786), (66906, 28576), (69683, 74216), (2171, 22531), (6591, 52559), (59362, 88732), (49865, 38016), (82881, 9151), (71356, 27365), (91463, 45944), (71040, 12264), (62116, 27681), (84941, 82284), (57515, 60713), (45227, 74196), (77502, 17765), (54055, 29484), (33708, 46220), (37344, 75102), (80480, 37942), (95782, 36781), (14092, 82457), (25284, 82297), (85150, 50659), (90037, 99122), (49584, 48321), (8028, 65210), (8162, 32091), (1609, 22211), (38548, 1023), (35864, 43021), (54196, 30754), (88535, 90185), (5518, 97226), (79272, 57758), (61381, 86419), (38344, 86877), (72629, 24694), (12563, 11747), (81456, 44023), (55536, 65195), (67741, 12182), (64250, 29636), (45252, 9834), (16576, 40098), (65924, 11123), (14090, 83443), (95180, 92697), (92452, 7957), (21244, 29717), (50253, 62030), (21401, 70735), (86803, 89625), (17066, 52757), (74570, 89660), (11647, 74336), (22463, 68191), (41693, 67542), (12994, 21955), (3180, 27966), (37236, 51099), (68568, 62778), (2671, 68542), (42771, 4268), (7578, 12919), (33838, 61136), (87904, 48076), (3781, 57164), (83289, 14414), (29626, 91409), (92536, 13946), (88125, 34961), (71541, 41249), (21045, 85183), (21628, 62294), (17768, 22390), (5892, 10231), (85902, 84414), (44078, 87944), (64981, 31018), (85690, 11077), (4711, 9309), (39077, 26339), (85313, 99603), (20267, 31090), (92684, 11726), (17887, 17415), (23064, 83141), (40925, 87246), (17575, 71065), (29396, 86192), (10859, 2881), (50457, 42309), (86972, 41824), (80954, 76970), (11664, 94322), (4286, 93562), (9093, 32589), (81261, 96519), (67886, 3086), (74132, 43197), (94549, 26192), (55498, 75374), (93813, 32911), (6727, 51629), (93543, 55373), (48682, 83316), (32468, 77339), (60028, 4896), (34177, 67214), (35429, 16686), (72814, 39944), (82791, 97364), (21249, 36007), (85536, 84815), (32784, 34349), (2391, 45661), (62595, 30434), (56513, 65051), (22536, 99421), (80603, 62636), (23609, 19082), (33262, 63428), (45307, 96049), (75079, 62835), (52361, 19400), (43845, 89033), (54257, 33298), (31133, 35836), (56725, 10158), (73650, 10456), (77548, 28249), (57998, 79546), (97895, 93196), (472, 18004), (67178, 2684), (34612, 36352), (84970, 98816), (32474, 5895), (60388, 75957), (81643, 26457), (30070, 55744), (14725, 58396), (66451, 69932), (59264, 59569), (98045, 63704), (11925, 87468), (24701, 6280), (47797, 84240), (998, 13575), (37865, 72036), (71339, 10339), (28248, 7481), (33945, 97829), (85194, 8904), (39669, 73192), (24326, 65934), (14438, 72729), (76676, 35176), (97604, 95931), (36032, 61056), (65788, 20291), (73067, 60718), (75742, 27362), (45393, 74792), (49586, 13048), (57961, 68534), (78171, 47271), (76064, 41030), (88738, 10155), (87757, 63134), (95783, 76438), (77935, 85529), (79345, 69555), (94483, 58668), (9432, 40485), (43585, 376), (61737, 97784), (1593, 37078), (11129, 23793), (94341, 64063), (41896, 70446), (77473, 33532), (66748, 65844), (17802, 27920), (22891, 12901), (40435, 43348), (27956, 16989), (29550, 63725), (49453, 61216), (25570, 97985), (39599, 83449), (96373, 12817), (48357, 40473), (99949, 33182), (85434, 45421), (28033, 46182), (80932, 26691), (99522, 29180), (70868, 31903), (63217, 51504), (70267, 29484), (36096, 28371), (45710, 2823), (85697, 3912), (25410, 70358), (73558, 82834), (28617, 80585), (81874, 57512), (76797, 45045), (22091, 84124), (68372, 42189), (66320, 83215), (23013, 22530), (33735, 97800), (54318, 99061), (18025, 39513), (69059, 39780), (58425, 39790), (10222, 96791), (17061, 3138), (92416, 5773), (20023, 31736), (47846, 74511), (52598, 26398), (3738, 44490), (53346, 79223), (31991, 57557), (34592, 98693), (86926, 20881), (14108, 87654), (4889, 34982), (77283, 73201), (13668, 22633), (50763, 95152), (97292, 54613), (22782, 30081), (21536, 46551), (45965, 4172), (75537, 79254), (41267, 86819), (57723, 75200), (60404, 27252), (80305, 57892), (50103, 24015), (69409, 13132), (30717, 56384), (54568, 79877), (8584, 8359), (41416, 64392), (32898, 92944), (5262, 35907), (88652, 77951), (16619, 2797), (42494, 65581), (21009, 18555), (29084, 28068), (29464, 35568), (7288, 52066), (32378, 48725), (95300, 62168), (52295, 97110), (36862, 87786), (12224, 86684), (85883, 34167), (45369, 76651), (95460, 42855), (34354, 87238), (7214, 96375), (58720, 87947), (55009, 10506), (98506, 6081), (46068, 81113), (99065, 58657), (81820, 28081), (90611, 57235), (7661, 72835), (93195, 90418), (34452, 20146), (39857, 78712), (47185, 52074), (9879, 51807), (92457, 92377), (36082, 17723), (21905, 43398), (35902, 54380), (21791, 6804), (50774, 35200), (11540, 83934), (54574, 57602), (99878, 35568), (28171, 68210), (58132, 17222), (91755, 80585), (18944, 65785), (78535, 70725), (42599, 65291), (62274, 97160), (76487, 23798), (34741, 36988), (7286, 61119), (36824, 29665), (82478, 60722), (90995, 28025), (71438, 40230), (83152, 45104), (1812, 91619)) - - SELECT (rand(), rand()) IN tuple(tuple(17258, 93148), tuple(4508, 52749), tuple(68660, 70017), tuple(77797, 23528), tuple(1136, 37393), tuple(53237, 15379), tuple(68370, 73211), tuple(15782, 54962), tuple(59432, 45415), tuple(68396, 920), tuple(96154, 21016), tuple(12700, 26887), tuple(88016, 43191), tuple(68153, 51575), tuple(91315, 40005), tuple(18070, 73178), tuple(86, 631), tuple(77717, 20324), tuple(3227, 76188), tuple(74960, 43147), tuple(77538, 19628), tuple(82292, 6525), tuple(24293, 12566), tuple(85244, 96287), tuple(93982, 1329), tuple(38064, 54723), tuple(83999, 45810), tuple(71921, 53673), tuple(88638, 9669), tuple(1959, 39535), tuple(82235, 95796), tuple(27907, 90975), tuple(42383, 91015), tuple(9948, 91514), tuple(81712, 47309), tuple(400, 25808), tuple(31791, 46948), tuple(39740, 36098), tuple(25943, 84598), tuple(99598, 52939), tuple(77134, 15845), tuple(40313, 72174), tuple(85017, 94036), tuple(36595, 14303), tuple(83961, 68078), tuple(55792, 72759), tuple(73574, 43606), tuple(9853, 63560), tuple(28580, 56721), tuple(74804, 41025), tuple(32095, 55657), tuple(52881, 63416), tuple(91368, 90310), tuple(23922, 38883), tuple(30592, 10758), tuple(66448, 61183), tuple(31880, 96697), tuple(11362, 20633), tuple(75331, 2015), tuple(71129, 8785), tuple(1115, 70955), tuple(7886, 83698), tuple(18961, 84556), tuple(16677, 43028), tuple(37347, 70220), tuple(31699, 71244), tuple(10578, 96159), tuple(67600, 39041), tuple(78791, 86687), tuple(21545, 54174), tuple(68774, 37637), tuple(46132, 81768), tuple(98413, 20605), tuple(2960, 23665), tuple(31507, 35719), tuple(96209, 18368), tuple(60558, 38035), tuple(21952, 3264), tuple(11834, 86458), tuple(21651, 17650), tuple(86276, 36087), tuple(18818, 24849), tuple(61951, 3390), tuple(59637, 62545), tuple(30346, 72253), tuple(36281, 2992), tuple(78340, 49872), tuple(94326, 93723), tuple(3416, 94405), tuple(12272, 8741), tuple(22600, 22095), tuple(57636, 37106), tuple(38702, 14889), tuple(70238, 11276), tuple(17325, 60648), tuple(16492, 41271), tuple(52100, 1304), tuple(93416, 7795), tuple(57209, 71008), tuple(48010, 36078), tuple(20384, 74420), tuple(77440, 34439), tuple(69224, 45099), tuple(30374, 33884), tuple(49038, 90140), tuple(1154, 84725), tuple(64926, 86985), tuple(91746, 73472), tuple(59757, 75755), tuple(45860, 71557), tuple(45833, 36526), tuple(74618, 73598), tuple(91360, 65168), tuple(58029, 30793), tuple(56332, 14973), tuple(99943, 96877), tuple(97454, 6450), tuple(64502, 77301), tuple(73182, 31853), tuple(76809, 83964), tuple(82916, 86188), tuple(78736, 65427), tuple(36495, 7422), tuple(76196, 2804), tuple(96117, 61093), tuple(9177, 26099), tuple(52942, 63007), tuple(48578, 47876), tuple(50638, 89903), tuple(7113, 97316), tuple(35301, 12750), tuple(47807, 7254), tuple(38217, 55418), tuple(56970, 41687), tuple(20527, 62886), tuple(358, 14021), tuple(64018, 18582), tuple(91740, 21683), tuple(81967, 53589), tuple(45437, 38450), tuple(45476, 67752), tuple(76851, 72072), tuple(7304, 60091), tuple(40097, 12897), tuple(39906, 29247), tuple(84262, 58734), tuple(30857, 43791), tuple(56087, 78929), tuple(20498, 45954), tuple(48726, 500), tuple(62723, 43763), tuple(28368, 30756), tuple(74048, 52403), tuple(15045, 95926), tuple(75542, 55384), tuple(52543, 22525), tuple(56001, 6935), tuple(11431, 46745), tuple(77731, 7310), tuple(36718, 59909), tuple(32235, 91254), tuple(92417, 25917), tuple(21782, 79277), tuple(46378, 87536), tuple(35324, 26075), tuple(6310, 76915), tuple(1551, 69473), tuple(50642, 68865), tuple(55190, 72934), tuple(49780, 21873), tuple(99466, 29686), tuple(90761, 13179), tuple(72959, 57033), tuple(20020, 90200), tuple(46186, 79105), tuple(73871, 52382), tuple(59559, 38801), tuple(59916, 16082), tuple(33610, 94966), tuple(46001, 45225), tuple(86679, 26469), tuple(77245, 91929), tuple(32887, 36623), tuple(11179, 46898), tuple(87881, 68087), tuple(45438, 47991), tuple(24950, 94525), tuple(91664, 51656), tuple(43914, 47805), tuple(15736, 96156), tuple(56346, 20283), tuple(85053, 48931), tuple(17790, 26179), tuple(96195, 55728), tuple(43765, 54807), tuple(44988, 89269), tuple(55911, 99411), tuple(52446, 47397), tuple(28346, 65442), tuple(96669, 68226), tuple(66194, 26848), tuple(37276, 55864), tuple(14116, 41583), tuple(18058, 16317), tuple(93136, 85318), tuple(35616, 86252), tuple(29222, 29969), tuple(33386, 85372), tuple(71094, 44238), tuple(27733, 31838), tuple(64626, 16692), tuple(52904, 97899), tuple(97619, 12663), tuple(50165, 4688), tuple(67557, 44053), tuple(69184, 66269), tuple(73164, 89705), tuple(39822, 15169), tuple(65499, 72808), tuple(30068, 63697), tuple(30154, 64235), tuple(97016, 58716), tuple(94366, 36592), tuple(1592, 16261), tuple(87985, 52102), tuple(12554, 23652), tuple(15909, 25292), tuple(2527, 91531), tuple(92139, 36031), tuple(28986, 30032), tuple(3038, 56314), tuple(32239, 26707), tuple(15973, 34901), tuple(70246, 39680), tuple(82529, 38132), tuple(45827, 74783), tuple(53665, 64111), tuple(55218, 84170), tuple(20466, 16130), tuple(55734, 71203), tuple(31438, 96906), tuple(66338, 85858), tuple(35988, 68511), tuple(78391, 15191), tuple(80747, 59213), tuple(5357, 11546), tuple(16822, 16607), tuple(36607, 41106), tuple(74949, 30739), tuple(45726, 64887), tuple(1524, 54847), tuple(37371, 89195), tuple(28726, 27788), tuple(22600, 44777), tuple(53999, 63625), tuple(84304, 98338), tuple(49260, 76480), tuple(74564, 53907), tuple(89867, 97096), tuple(60157, 61299), tuple(17165, 10146), tuple(56334, 36268), tuple(62114, 49222), tuple(22715, 23620), tuple(42830, 11539), tuple(41091, 69151), tuple(75471, 68364), tuple(18681, 43249), tuple(42738, 63219), tuple(35474, 98454), tuple(76815, 46024), tuple(66310, 36521), tuple(86095, 77013), tuple(63693, 77319), tuple(80731, 63031), tuple(95478, 92387), tuple(23787, 63724), tuple(46299, 68994), tuple(4800, 2460), tuple(9663, 80639), tuple(77231, 85814), tuple(81615, 11311), tuple(35638, 27340), tuple(13598, 14322), tuple(30657, 17238), tuple(90957, 96846), tuple(69962, 52140), tuple(41681, 65962), tuple(96836, 58177), tuple(36190, 11623), tuple(4231, 40500), tuple(43049, 41949), tuple(71177, 98492), tuple(30193, 39750), tuple(19744, 33204), tuple(63358, 30210), tuple(45638, 58918), tuple(43641, 38741), tuple(35598, 40932), tuple(33238, 36236), tuple(50835, 20968), tuple(25099, 34071), tuple(84986, 88456), tuple(35333, 1529), tuple(79771, 23985), tuple(647, 61658), tuple(9424, 11743), tuple(77766, 31528), tuple(77811, 86973), tuple(76403, 74377), tuple(55568, 79251), tuple(68858, 20762), tuple(68520, 66773), tuple(93598, 89823), tuple(8080, 82539), tuple(87760, 52247), tuple(25191, 16905), tuple(17837, 8339), tuple(85177, 59050), tuple(51680, 77374), tuple(3287, 43018), tuple(43479, 62141), tuple(34909, 46322), tuple(11869, 5885), tuple(96193, 58417), tuple(101, 47460), tuple(34937, 88582), tuple(83216, 88388), tuple(28571, 15292), tuple(66683, 62613), tuple(34478, 8924), tuple(2680, 89973), tuple(62438, 44460), tuple(11724, 4791), tuple(5383, 72888), tuple(88206, 67586), tuple(8124, 21690), tuple(28779, 75789), tuple(66791, 4757), tuple(6176, 47760), tuple(6403, 78084), tuple(78122, 35446), tuple(99494, 73608), tuple(39691, 89098), tuple(59182, 19484), tuple(25389, 98963), tuple(96487, 3692), tuple(76222, 67381), tuple(21199, 50358), tuple(95998, 58137), tuple(28777, 43913), tuple(14176, 60117), tuple(52257, 81703), tuple(14604, 13438), tuple(71301, 14401), tuple(19758, 66914), tuple(15506, 29873), tuple(87205, 29449), tuple(93295, 15930), tuple(63651, 11287), tuple(19785, 15966), tuple(30795, 75112), tuple(69462, 37655), tuple(18793, 85764), tuple(36240, 31236), tuple(98153, 73724), tuple(72491, 4223), tuple(66930, 35048), tuple(25686, 13269), tuple(13940, 13259), tuple(69163, 11235), tuple(1183, 86961), tuple(54323, 67315), tuple(85044, 60872), tuple(48875, 3683), tuple(43052, 92861), tuple(87574, 32969), tuple(92552, 80564), tuple(94832, 47682), tuple(72011, 80994), tuple(60182, 917), tuple(97788, 34169), tuple(66432, 47940), tuple(87468, 80954), tuple(35385, 68758), tuple(50555, 63710), tuple(55311, 44337), tuple(87065, 26514), tuple(84581, 98736), tuple(23212, 56499), tuple(75120, 72447), tuple(56087, 38285), tuple(58171, 45629), tuple(28401, 44319), tuple(70432, 27883), tuple(18891, 14646), tuple(26206, 49924), tuple(79957, 44914), tuple(56064, 27529), tuple(99090, 29197), tuple(49435, 340), tuple(53525, 65601), tuple(76998, 88349), tuple(50416, 70860), tuple(42506, 75290), tuple(34024, 13295), tuple(86663, 46523), tuple(88814, 231), tuple(57809, 21), tuple(84914, 84771), tuple(43042, 66892), tuple(17288, 33908), tuple(4934, 63195), tuple(50590, 1516), tuple(97843, 80208), tuple(20091, 86717), tuple(71566, 15929), tuple(19531, 23634), tuple(41646, 45549), tuple(89226, 82902), tuple(96683, 63386), tuple(31072, 53788), tuple(51135, 41099), tuple(78912, 65609), tuple(36094, 23603), tuple(88403, 51455), tuple(73795, 47066), tuple(26448, 82852), tuple(22829, 2894), tuple(30041, 92548), tuple(27733, 20608), tuple(70180, 19892), tuple(51650, 63440), tuple(76328, 13666), tuple(40514, 6677), tuple(2786, 51059), tuple(40809, 16499), tuple(10857, 82541), tuple(78221, 61067), tuple(17982, 51969), tuple(85369, 66965), tuple(47153, 47149), tuple(43965, 75796), tuple(82725, 60767), tuple(42407, 97249), tuple(51475, 81224), tuple(60957, 89414), tuple(33065, 21663), tuple(36601, 5290), tuple(95842, 67301), tuple(64630, 60398), tuple(55212, 35638), tuple(41750, 44235), tuple(75260, 82400), tuple(91291, 25843), tuple(6477, 8311), tuple(14919, 52306), tuple(66220, 33180), tuple(45736, 2313), tuple(37450, 64444), tuple(98614, 61344), tuple(75007, 50946), tuple(56701, 28117), tuple(66632, 5174), tuple(92323, 76613), tuple(6796, 73695), tuple(33696, 76280), tuple(86876, 5614), tuple(50863, 67993), tuple(36068, 17049), tuple(91912, 34271), tuple(70706, 1904), tuple(97798, 41117), tuple(68154, 72483), tuple(83862, 25578), tuple(61643, 17204), tuple(69974, 64232), tuple(77926, 19637), + + SELECT (rand(), rand()) IN tuple(tuple(17258, 93148), tuple(4508, 52749), tuple(68660, 70017), tuple(77797, 23528), tuple(1136, 37393), tuple(53237, 15379), tuple(68370, 73211), tuple(15782, 54962), tuple(59432, 45415), tuple(68396, 920), tuple(96154, 21016), tuple(12700, 26887), tuple(88016, 43191), tuple(68153, 51575), tuple(91315, 40005), tuple(18070, 73178), tuple(86, 631), tuple(77717, 20324), tuple(3227, 76188), tuple(74960, 43147), tuple(77538, 19628), tuple(82292, 6525), tuple(24293, 12566), tuple(85244, 96287), tuple(93982, 1329), tuple(38064, 54723), tuple(83999, 45810), tuple(71921, 53673), tuple(88638, 9669), tuple(1959, 39535), tuple(82235, 95796), tuple(27907, 90975), tuple(42383, 91015), tuple(9948, 91514), tuple(81712, 47309), tuple(400, 25808), tuple(31791, 46948), tuple(39740, 36098), tuple(25943, 84598), tuple(99598, 52939), tuple(77134, 15845), tuple(40313, 72174), tuple(85017, 94036), tuple(36595, 14303), tuple(83961, 68078), tuple(55792, 72759), tuple(73574, 43606), tuple(9853, 63560), tuple(28580, 56721), tuple(74804, 41025), tuple(32095, 55657), tuple(52881, 63416), tuple(91368, 90310), tuple(23922, 38883), tuple(30592, 10758), tuple(66448, 61183), tuple(31880, 96697), tuple(11362, 20633), tuple(75331, 2015), tuple(71129, 8785), tuple(1115, 70955), tuple(7886, 83698), tuple(18961, 84556), tuple(16677, 43028), tuple(37347, 70220), tuple(31699, 71244), tuple(10578, 96159), tuple(67600, 39041), tuple(78791, 86687), tuple(21545, 54174), tuple(68774, 37637), tuple(46132, 81768), tuple(98413, 20605), tuple(2960, 23665), tuple(31507, 35719), tuple(96209, 18368), tuple(60558, 38035), tuple(21952, 3264), tuple(11834, 86458), tuple(21651, 17650), tuple(86276, 36087), tuple(18818, 24849), tuple(61951, 3390), tuple(59637, 62545), tuple(30346, 72253), tuple(36281, 2992), tuple(78340, 49872), tuple(94326, 93723), tuple(3416, 94405), tuple(12272, 8741), tuple(22600, 22095), tuple(57636, 37106), tuple(38702, 14889), tuple(70238, 11276), tuple(17325, 60648), tuple(16492, 41271), tuple(52100, 1304), tuple(93416, 7795), tuple(57209, 71008), tuple(48010, 36078), tuple(20384, 74420), tuple(77440, 34439), tuple(69224, 45099), tuple(30374, 33884), tuple(49038, 90140), tuple(1154, 84725), tuple(64926, 86985), tuple(91746, 73472), tuple(59757, 75755), tuple(45860, 71557), tuple(45833, 36526), tuple(74618, 73598), tuple(91360, 65168), tuple(58029, 30793), tuple(56332, 14973), tuple(99943, 96877), tuple(97454, 6450), tuple(64502, 77301), tuple(73182, 31853), tuple(76809, 83964), tuple(82916, 86188), tuple(78736, 65427), tuple(36495, 7422), tuple(76196, 2804), tuple(96117, 61093), tuple(9177, 26099), tuple(52942, 63007), tuple(48578, 47876), tuple(50638, 89903), tuple(7113, 97316), tuple(35301, 12750), tuple(47807, 7254), tuple(38217, 55418), tuple(56970, 41687), tuple(20527, 62886), tuple(358, 14021), tuple(64018, 18582), tuple(91740, 21683), tuple(81967, 53589), tuple(45437, 38450), tuple(45476, 67752), tuple(76851, 72072), tuple(7304, 60091), tuple(40097, 12897), tuple(39906, 29247), tuple(84262, 58734), tuple(30857, 43791), tuple(56087, 78929), tuple(20498, 45954), tuple(48726, 500), tuple(62723, 43763), tuple(28368, 30756), tuple(74048, 52403), tuple(15045, 95926), tuple(75542, 55384), tuple(52543, 22525), tuple(56001, 6935), tuple(11431, 46745), tuple(77731, 7310), tuple(36718, 59909), tuple(32235, 91254), tuple(92417, 25917), tuple(21782, 79277), tuple(46378, 87536), tuple(35324, 26075), tuple(6310, 76915), tuple(1551, 69473), tuple(50642, 68865), tuple(55190, 72934), tuple(49780, 21873), tuple(99466, 29686), tuple(90761, 13179), tuple(72959, 57033), tuple(20020, 90200), tuple(46186, 79105), tuple(73871, 52382), tuple(59559, 38801), tuple(59916, 16082), tuple(33610, 94966), tuple(46001, 45225), tuple(86679, 26469), tuple(77245, 91929), tuple(32887, 36623), tuple(11179, 46898), tuple(87881, 68087), tuple(45438, 47991), tuple(24950, 94525), tuple(91664, 51656), tuple(43914, 47805), tuple(15736, 96156), tuple(56346, 20283), tuple(85053, 48931), tuple(17790, 26179), tuple(96195, 55728), tuple(43765, 54807), tuple(44988, 89269), tuple(55911, 99411), tuple(52446, 47397), tuple(28346, 65442), tuple(96669, 68226), tuple(66194, 26848), tuple(37276, 55864), tuple(14116, 41583), tuple(18058, 16317), tuple(93136, 85318), tuple(35616, 86252), tuple(29222, 29969), tuple(33386, 85372), tuple(71094, 44238), tuple(27733, 31838), tuple(64626, 16692), tuple(52904, 97899), tuple(97619, 12663), tuple(50165, 4688), tuple(67557, 44053), tuple(69184, 66269), tuple(73164, 89705), tuple(39822, 15169), tuple(65499, 72808), tuple(30068, 63697), tuple(30154, 64235), tuple(97016, 58716), tuple(94366, 36592), tuple(1592, 16261), tuple(87985, 52102), tuple(12554, 23652), tuple(15909, 25292), tuple(2527, 91531), tuple(92139, 36031), tuple(28986, 30032), tuple(3038, 56314), tuple(32239, 26707), tuple(15973, 34901), tuple(70246, 39680), tuple(82529, 38132), tuple(45827, 74783), tuple(53665, 64111), tuple(55218, 84170), tuple(20466, 16130), tuple(55734, 71203), tuple(31438, 96906), tuple(66338, 85858), tuple(35988, 68511), tuple(78391, 15191), tuple(80747, 59213), tuple(5357, 11546), tuple(16822, 16607), tuple(36607, 41106), tuple(74949, 30739), tuple(45726, 64887), tuple(1524, 54847), tuple(37371, 89195), tuple(28726, 27788), tuple(22600, 44777), tuple(53999, 63625), tuple(84304, 98338), tuple(49260, 76480), tuple(74564, 53907), tuple(89867, 97096), tuple(60157, 61299), tuple(17165, 10146), tuple(56334, 36268), tuple(62114, 49222), tuple(22715, 23620), tuple(42830, 11539), tuple(41091, 69151), tuple(75471, 68364), tuple(18681, 43249), tuple(42738, 63219), tuple(35474, 98454), tuple(76815, 46024), tuple(66310, 36521), tuple(86095, 77013), tuple(63693, 77319), tuple(80731, 63031), tuple(95478, 92387), tuple(23787, 63724), tuple(46299, 68994), tuple(4800, 2460), tuple(9663, 80639), tuple(77231, 85814), tuple(81615, 11311), tuple(35638, 27340), tuple(13598, 14322), tuple(30657, 17238), tuple(90957, 96846), tuple(69962, 52140), tuple(41681, 65962), tuple(96836, 58177), tuple(36190, 11623), tuple(4231, 40500), tuple(43049, 41949), tuple(71177, 98492), tuple(30193, 39750), tuple(19744, 33204), tuple(63358, 30210), tuple(45638, 58918), tuple(43641, 38741), tuple(35598, 40932), tuple(33238, 36236), tuple(50835, 20968), tuple(25099, 34071), tuple(84986, 88456), tuple(35333, 1529), tuple(79771, 23985), tuple(647, 61658), tuple(9424, 11743), tuple(77766, 31528), tuple(77811, 86973), tuple(76403, 74377), tuple(55568, 79251), tuple(68858, 20762), tuple(68520, 66773), tuple(93598, 89823), tuple(8080, 82539), tuple(87760, 52247), tuple(25191, 16905), tuple(17837, 8339), tuple(85177, 59050), tuple(51680, 77374), tuple(3287, 43018), tuple(43479, 62141), tuple(34909, 46322), tuple(11869, 5885), tuple(96193, 58417), tuple(101, 47460), tuple(34937, 88582), tuple(83216, 88388), tuple(28571, 15292), tuple(66683, 62613), tuple(34478, 8924), tuple(2680, 89973), tuple(62438, 44460), tuple(11724, 4791), tuple(5383, 72888), tuple(88206, 67586), tuple(8124, 21690), tuple(28779, 75789), tuple(66791, 4757), tuple(6176, 47760), tuple(6403, 78084), tuple(78122, 35446), tuple(99494, 73608), tuple(39691, 89098), tuple(59182, 19484), tuple(25389, 98963), tuple(96487, 3692), tuple(76222, 67381), tuple(21199, 50358), tuple(95998, 58137), tuple(28777, 43913), tuple(14176, 60117), tuple(52257, 81703), tuple(14604, 13438), tuple(71301, 14401), tuple(19758, 66914), tuple(15506, 29873), tuple(87205, 29449), tuple(93295, 15930), tuple(63651, 11287), tuple(19785, 15966), tuple(30795, 75112), tuple(69462, 37655), tuple(18793, 85764), tuple(36240, 31236), tuple(98153, 73724), tuple(72491, 4223), tuple(66930, 35048), tuple(25686, 13269), tuple(13940, 13259), tuple(69163, 11235), tuple(1183, 86961), tuple(54323, 67315), tuple(85044, 60872), tuple(48875, 3683), tuple(43052, 92861), tuple(87574, 32969), tuple(92552, 80564), tuple(94832, 47682), tuple(72011, 80994), tuple(60182, 917), tuple(97788, 34169), tuple(66432, 47940), tuple(87468, 80954), tuple(35385, 68758), tuple(50555, 63710), tuple(55311, 44337), tuple(87065, 26514), tuple(84581, 98736), tuple(23212, 56499), tuple(75120, 72447), tuple(56087, 38285), tuple(58171, 45629), tuple(28401, 44319), tuple(70432, 27883), tuple(18891, 14646), tuple(26206, 49924), tuple(79957, 44914), tuple(56064, 27529), tuple(99090, 29197), tuple(49435, 340), tuple(53525, 65601), tuple(76998, 88349), tuple(50416, 70860), tuple(42506, 75290), tuple(34024, 13295), tuple(86663, 46523), tuple(88814, 231), tuple(57809, 21), tuple(84914, 84771), tuple(43042, 66892), tuple(17288, 33908), tuple(4934, 63195), tuple(50590, 1516), tuple(97843, 80208), tuple(20091, 86717), tuple(71566, 15929), tuple(19531, 23634), tuple(41646, 45549), tuple(89226, 82902), tuple(96683, 63386), tuple(31072, 53788), tuple(51135, 41099), tuple(78912, 65609), tuple(36094, 23603), tuple(88403, 51455), tuple(73795, 47066), tuple(26448, 82852), tuple(22829, 2894), tuple(30041, 92548), tuple(27733, 20608), tuple(70180, 19892), tuple(51650, 63440), tuple(76328, 13666), tuple(40514, 6677), tuple(2786, 51059), tuple(40809, 16499), tuple(10857, 82541), tuple(78221, 61067), tuple(17982, 51969), tuple(85369, 66965), tuple(47153, 47149), tuple(43965, 75796), tuple(82725, 60767), tuple(42407, 97249), tuple(51475, 81224), tuple(60957, 89414), tuple(33065, 21663), tuple(36601, 5290), tuple(95842, 67301), tuple(64630, 60398), tuple(55212, 35638), tuple(41750, 44235), tuple(75260, 82400), tuple(91291, 25843), tuple(6477, 8311), tuple(14919, 52306), tuple(66220, 33180), tuple(45736, 2313), tuple(37450, 64444), tuple(98614, 61344), tuple(75007, 50946), tuple(56701, 28117), tuple(66632, 5174), tuple(92323, 76613), tuple(6796, 73695), tuple(33696, 76280), tuple(86876, 5614), tuple(50863, 67993), tuple(36068, 17049), tuple(91912, 34271), tuple(70706, 1904), tuple(97798, 41117), tuple(68154, 72483), tuple(83862, 25578), tuple(61643, 17204), tuple(69974, 64232), tuple(77926, 19637), tuple(64901, 88988), tuple(71424, 91703), tuple(91655, 17147), tuple(46872, 56530), tuple(44189, 98087), tuple(95939, 54420), tuple(72651, 68785), tuple(67624, 84875), tuple(92587, 87663), tuple(65275, 81256), tuple(53798, 2506), tuple(14702, 3638), tuple(71291, 50452), tuple(14909, 13903), tuple(66965, 26606), tuple(14127, 60345), tuple(35306, 1738), tuple(77234, 10468), tuple(53521, 41218), tuple(80681, 82583), tuple(44227, 26521), tuple(32263, 21482), tuple(82270, 56963), tuple(50580, 80567), tuple(11593, 22346), tuple(20074, 26867), tuple(73126, 28667), tuple(62996, 24317), tuple(20295, 57163), tuple(1506, 57668), tuple(69567, 45236), tuple(43366, 26001), tuple(88052, 40181), tuple(1599, 89349), tuple(36789, 1579), tuple(39895, 46673), tuple(30381, 3206), tuple(31723, 5625), tuple(19252, 31317), tuple(16932, 77149), tuple(48794, 34409), tuple(55986, 30328), tuple(47551, 75088), tuple(57363, 78365), tuple(95221, 63385), tuple(26449, 5733), tuple(96588, 53077), tuple(52980, 41140), tuple(8187, 85947), tuple(36723, 26520), tuple(23579, 38909), tuple(33350, 19275), tuple(63930, 19357), tuple(43536, 59941), tuple(31117, 77322), tuple(44638, 94812), tuple(44730, 99097), tuple(95108, 48170), tuple(57813, 49503), tuple(79959, 89436), tuple(86980, 62031), tuple(8275, 44009), tuple(36666, 94645), tuple(22064, 38882), tuple(40471, 16939), tuple(31156, 11337), tuple(13101, 96977), tuple(17906, 26835), tuple(89861, 51405), tuple(73369, 67946), tuple(99141, 58572), tuple(27131, 98703), tuple(15900, 43412), tuple(51768, 93125), tuple(78579, 46689), tuple(23029, 13895), tuple(60870, 55830), tuple(22553, 8236), tuple(76449, 96207), tuple(83766, 51024), tuple(27630, 50614), tuple(53484, 90104), tuple(77626, 21944), tuple(46755, 41583), tuple(53616, 34240), tuple(94159, 44415), tuple(13914, 90059), tuple(44387, 89012), tuple(27499, 64579), tuple(83415, 30809), tuple(77558, 82619), tuple(88880, 9814), tuple(8466, 4424), tuple(43598, 91921), tuple(24695, 3349), tuple(46295, 65208), tuple(51256, 82461), tuple(49126, 93012), tuple(16186, 96585), tuple(43284, 22655), tuple(93130, 90393), tuple(77495, 34372), tuple(85509, 65856), tuple(86662, 61906), tuple(50988, 44393), tuple(29828, 17737), tuple(91651, 35308), tuple(29796, 49716), tuple(14019, 87751), tuple(29688, 71207), tuple(82845, 19100), tuple(11989, 50132), tuple(21158, 99905), tuple(54732, 42547), tuple(32314, 12851), tuple(46405, 43794), tuple(87849, 45643), tuple(53524, 21212), tuple(61925, 75491), tuple(12498, 21937), tuple(30185, 69475), tuple(48421, 52487), tuple(15112, 90935), tuple(33187, 17801), tuple(61704, 25514), tuple(17889, 23917), tuple(18758, 57197), tuple(7693, 47232), tuple(47905, 24618), tuple(11494, 78950), tuple(95662, 54561), tuple(8075, 33909), tuple(90427, 46065), tuple(73962, 19821), tuple(50691, 79400), tuple(58218, 4881), tuple(94106, 2509), tuple(60633, 55169), tuple(49600, 83054), tuple(23339, 13270), tuple(70262, 58946), tuple(48417, 97266), tuple(27629, 46905), tuple(74465, 75514), tuple(41687, 2564), tuple(12814, 19492), tuple(78899, 30168), tuple(17745, 35206), tuple(37972, 35296), tuple(22288, 80001), tuple(68026, 36558), tuple(40187, 12234), tuple(92380, 22866), tuple(56488, 64402), tuple(41404, 62562), tuple(47802, 45287), tuple(83302, 85215), tuple(58999, 85776), tuple(35158, 16804), tuple(13416, 94146), tuple(62953, 28243), tuple(83290, 19103), tuple(4564, 21789), tuple(64468, 20927), tuple(25582, 47206), tuple(57810, 18693), tuple(28938, 97986), tuple(61704, 14838), tuple(19214, 3232), tuple(12911, 25438), tuple(85802, 28837), tuple(56506, 89458), tuple(66392, 47773), tuple(68190, 43841), tuple(43044, 52214), tuple(57886, 32830), tuple(15943, 59771), tuple(37081, 89294), tuple(4032, 32960), tuple(46931, 85790), tuple(69656, 72737), tuple(28217, 39872), tuple(86170, 42776), tuple(55116, 51495), tuple(90485, 45274), tuple(60773, 36788), tuple(2193, 2636), tuple(70222, 62086), tuple(75720, 70712), tuple(17549, 51460), tuple(23609, 31515), tuple(70254, 39825), tuple(63762, 11061), tuple(13107, 15394), tuple(45916, 72130), tuple(91558, 86662), tuple(99524, 69106), tuple(93073, 29881), tuple(31724, 3007), tuple(69051, 59452), tuple(59701, 86760), tuple(4967, 82028), tuple(57404, 48226), tuple(71829, 79910), tuple(23714, 62439), tuple(73881, 67618), tuple(63269, 40085), tuple(6164, 23415), tuple(48156, 93907), tuple(18627, 16570), tuple(6676, 22991), tuple(36916, 41488), tuple(99079, 13264), tuple(32533, 99243), tuple(55505, 63339), tuple(89564, 3290), tuple(24886, 34916), tuple(91310, 9343), tuple(49779, 12740), tuple(26320, 3406), tuple(57661, 5702), tuple(10765, 57881), tuple(5518, 47638), tuple(93148, 27438), tuple(73451, 24477), tuple(84075, 96822), tuple(58883, 58883), tuple(96812, 82388), tuple(30659, 59654), tuple(24498, 95808), tuple(25591, 21834), tuple(13090, 87704), tuple(76495, 17249), tuple(75975, 84318), tuple(55459, 70426), tuple(84256, 88604), tuple(79438, 43104), tuple(45331, 7495), tuple(63619, 11123), tuple(24772, 2601), tuple(63343, 14138), tuple(39957, 98339), tuple(55595, 17823), tuple(97676, 53933), tuple(91867, 25023), tuple(64677, 67859), tuple(43737, 34315), tuple(24800, 53968), tuple(93157, 17507), tuple(24264, 35273), tuple(33889, 507), tuple(10207, 40542), tuple(40213, 57800), tuple(38321, 74160), tuple(42391, 7651), tuple(80267, 94736), tuple(52473, 79634), tuple(17075, 2531), tuple(8595, 75890), tuple(31496, 50367), tuple(16069, 79896), tuple(70067, 200), tuple(23420, 49517), tuple(1628, 45646), tuple(8916, 36794), tuple(72294, 88976), tuple(40603, 86008), tuple(91871, 71098), tuple(5447, 70998), tuple(24152, 17561), tuple(65046, 34951), tuple(56950, 9292), tuple(19244, 31385), tuple(74693, 31813), tuple(97343, 21572), tuple(38834, 135), tuple(79717, 62486), tuple(38, 10308), tuple(58035, 71344), tuple(85802, 81079), tuple(5943, 156), tuple(38735, 38867), tuple(3803, 99366), tuple(15853, 19408), tuple(62988, 62008), tuple(8316, 44684), tuple(17035, 71012), tuple(48584, 2117), tuple(75425, 37336), tuple(2405, 50420), tuple(43653, 28836), tuple(12394, 69430), tuple(54522, 4954), tuple(33359, 148), tuple(41018, 82851), tuple(79995, 55417), tuple(65008, 32342), tuple(36547, 88185), tuple(8131, 7054), tuple(38980, 20146), tuple(27976, 63039), tuple(53119, 67009), tuple(40043, 98393), tuple(29333, 51980), tuple(85818, 98405), tuple(77956, 20099), tuple(99747, 16916), tuple(11597, 50181), tuple(40961, 8262), tuple(75103, 13912), tuple(62339, 69155), tuple(3869, 85481), tuple(7053, 30956), tuple(33563, 53272), tuple(96178, 81751), tuple(99365, 88728), tuple(34447, 11164), tuple(62856, 30939), tuple(92486, 3357), tuple(56605, 35330), tuple(42180, 15137), tuple(83946, 62984), tuple(61869, 55711), tuple(52880, 49871), tuple(44588, 27387), tuple(16332, 24496), tuple(1781, 13508), tuple(56674, 95773), tuple(21328, 19628), tuple(96455, 24155), tuple(14302, 74435), tuple(54053, 24590), tuple(86642, 22177), tuple(24089, 16186), tuple(70281, 4601), tuple(18552, 70708), tuple(95442, 5895), tuple(96714, 6293), tuple(43803, 45857), tuple(93257, 18497), tuple(90032, 85086), tuple(40566, 87233), tuple(32674, 73822), tuple(95599, 49334), tuple(62745, 51898), tuple(8245, 93882), tuple(14093, 40977), tuple(47215, 53001), tuple(59737, 68452), tuple(90937, 25354), tuple(43805, 82571), tuple(81953, 68572), tuple(37298, 96262), tuple(94899, 65066), tuple(34772, 80762), tuple(55469, 1186), tuple(8734, 91665), tuple(18622, 51150), tuple(85200, 39575), tuple(65381, 15979), tuple(89734, 89656), tuple(64712, 53691), tuple(87187, 58256), tuple(8476, 89694), tuple(49935, 35239), tuple(63730, 34982), tuple(27687, 91571), tuple(87543, 15350), tuple(85208, 18781), tuple(14783, 2574), tuple(44699, 666), tuple(56440, 87617), tuple(32732, 49301), tuple(76725, 3895), tuple(10419, 90580), tuple(34725, 69476), tuple(14831, 81588), tuple(93924, 38057), tuple(38528, 99060), tuple(57136, 44206), tuple(74685, 99559), tuple(43083, 87511), tuple(43105, 35474), tuple(35582, 17560), tuple(5578, 98727), tuple(78947, 53865), tuple(32013, 95029), tuple(61552, 42674), tuple(52191, 49975), tuple(71566, 16403), tuple(78534, 16350), tuple(18520, 80501), tuple(29114, 46547), tuple(11488, 5069), tuple(89591, 82384), tuple(13741, 42318), tuple(74385, 58849), tuple(49739, 63421), tuple(83821, 6676), tuple(51997, 93321), tuple(36677, 81768), tuple(37915, 73495), tuple(47175, 6086), tuple(39989, 83110), tuple(6489, 48112), tuple(88822, 20370), tuple(12846, 13952), tuple(28930, 20879), tuple(25139, 84552), tuple(76434, 2665), tuple(55145, 31523), tuple(21177, 18630), tuple(81077, 96275), tuple(61006, 30845), tuple(77722, 62651), tuple(61181, 72545), tuple(93838, 84287), tuple(59300, 19014), tuple(75076, 97980), tuple(76979, 1473), tuple(48409, 13097), tuple(51718, 5325), tuple(36522, 72119), tuple(60917, 18995), tuple(61469, 42853), tuple(34387, 37322), tuple(38684, 28120), tuple(64136, 8559), tuple(15368, 99424), tuple(97824, 7864), tuple(33833, 72029), tuple(7024, 9961), tuple(49400, 66220), tuple(63025, 97179), tuple(6135, 98878), tuple(19873, 8438), tuple(3963, 35670), tuple(65186, 89423), tuple(26653, 65943), tuple(83132, 67000), tuple(82578, 35007), tuple(42680, 60479), tuple(71102, 98589), tuple(74842, 94010), tuple(22931, 33725), tuple(46537, 42629), tuple(75793, 48115), tuple(21630, 92454), tuple(97993, 81332), tuple(25747, 31814), tuple(91231, 65953), tuple(91981, 12219), tuple(64719, 16254), tuple(60914, 8334), tuple(15887, 96432), tuple(42110, 28837), tuple(7295, 83147), tuple(50334, 7053), tuple(3949, 33594), tuple(1524, 98230), tuple(17265, 98024), tuple(75969, 36232), tuple(89538, 5212), tuple(13444, 55946), tuple(69823, 81848), tuple(32578, 74024), tuple(52018, 98290), tuple(59118, 40186), tuple(61002, 16977), tuple(69537, 44780), tuple(92, 13937), tuple(33715, 42663), tuple(46347, 8312), tuple(86196, 59301), tuple(17128, 85014), tuple(26429, 57682), tuple(45888, 99588), tuple(22750, 96110), tuple(46809, 49251), tuple(24521, 40071), tuple(287, 22115), tuple(11741, 36315), tuple(22742, 17581), tuple(35808, 3110), tuple(98904, 30407), tuple(4584, 13383), tuple(28585, 69669), tuple(94823, 29715), tuple(9551, 36389), tuple(77997, 45746), tuple(49894, 55722), tuple(23415, 69459), tuple(58246, 85144), tuple(74136, 18102), tuple(97366, 85724), tuple(34271, 51601), tuple(47535, 70883), tuple(59443, 90103), tuple(45213, 45811), tuple(62741, 86898), tuple(17324, 50034), tuple(62080, 25193), tuple(89524, 4421), tuple(13476, 51456), tuple(69198, 56718), tuple(58024, 22969), tuple(65210, 67941), tuple(32561, 44881), tuple(62295, 67448), tuple(66135, 95453), tuple(9417, 20443), tuple(82486, 23745), tuple(19185, 99041), tuple(40662, 91714), tuple(3423, 58624), tuple(4512, 74502), tuple(67772, 98023), tuple(69575, 75779), tuple(69107, 62805), tuple(517, 33801), tuple(47406, 7581), tuple(81108, 10546), tuple(12976, 47001), tuple(16742, 83811), tuple(44593, 82124), tuple(52731, 34642), tuple(81725, 20555), tuple(94126, 91919), tuple(24800, 59302), tuple(97253, 39249), tuple(71692, 10769), tuple(88721, 56321), tuple(7019, 69771), tuple(31464, 61774), tuple(29597, 19263), tuple(65557, 31875), tuple(28653, 69636), tuple(58074, 76848), tuple(15906, 80620), tuple(18259, 40193), tuple(99991, 4769), tuple(98935, 99269), tuple(12123, 60124), tuple(20787, 47346), tuple(13526, 33592), tuple(95370, 40350), tuple(17479, 42884), tuple(58368, 83218), tuple(63290, 74406), tuple(97030, 35102), tuple(45298, 27660), tuple(64593, 21262), tuple(76268, 82641), tuple(1107, 44044), tuple(21427, 79959), tuple(85180, 62412), tuple(7359, 1318), tuple(83618, 9762), tuple(1425, 55804), tuple(32874, 97943), tuple(68191, 38742), tuple(41715, 17902), tuple(3771, 15032), tuple(7848, 74950), tuple(33881, 40904), tuple(75295, 26151), tuple(75775, 13760), tuple(90262, 89822), tuple(88169, 18679), tuple(57506, 32356), tuple(94983, 44281), tuple(37385, 37432), tuple(18248, 48162), tuple(45573, 66278), tuple(25277, 72788), tuple(26977, 36778), tuple(26254, 61758), tuple(12860, 48026), tuple(96819, 3339), tuple(13134, 1173), tuple(26822, 53374), tuple(15989, 29698), tuple(11258, 54515), tuple(37866, 34928), tuple(22996, 26577), tuple(39952, 42732), tuple(6754, 70595), tuple(86245, 44669), tuple(47044, 34170), tuple(6789, 45220), tuple(31706, 2090), tuple(42582, 40023), tuple(35147, 46591), tuple(88210, 11307), tuple(53644, 7680), tuple(11280, 91075), tuple(42961, 65122), tuple(40066, 52185), tuple(20050, 6154), tuple(98440, 20393), tuple(88992, 75432), tuple(32386, 66731), tuple(36952, 34149), tuple(18453, 32715), tuple(84413, 10378), tuple(59440, 2374), tuple(45354, 85009), tuple(50382, 66510), tuple(64428, 95401), tuple(9336, 41760), tuple(26317, 91416), tuple(81941, 99504), tuple(26600, 53522), tuple(81069, 40236), tuple(51126, 27911), tuple(97144, 14243), tuple(62738, 50287), tuple(37372, 28962), tuple(12053, 9090), tuple(69492, 95524), tuple(68141, 52931), tuple(17276, 16487), tuple(69227, 25949), tuple(14143, 70193), tuple(7077, 53032), tuple(65463, 74082), tuple(94997, 66496), tuple(80443, 55832), tuple(66796, 5970), tuple(15852, 95662), tuple(81559, 97272), tuple(55851, 18977), tuple(91142, 48976), tuple(91143, 950), tuple(79225, 31004), tuple(61310, 20760), tuple(74541, 90842), tuple(80322, 11630), tuple(84631, 544), tuple(66785, 86591), tuple(25650, 63252), tuple(59635, 18586), tuple(2964, 6741), tuple(37091, 71148), tuple(11984, 43077), tuple(87505, 62049), tuple(61925, 92290), tuple(18808, 3937), tuple(8300, 33268), tuple(70850, 50661), tuple(86024, 73730), tuple(85161, 47116), tuple(50193, 89155), tuple(37773, 40845), tuple(9251, 41688), tuple(6940, 65399), tuple(42479, 95630), tuple(19401, 43102), tuple(48069, 36040), tuple(62760, 95013), tuple(394, 2641), tuple(32567, 29306), tuple(13870, 58835), tuple(98248, 47291), tuple(49803, 4523), tuple(40222, 12883), tuple(53576, 73105), tuple(88265, 23629), tuple(67865, 67875), tuple(33473, 27144), tuple(80219, 53893), tuple(74878, 47341), tuple(78070, 84803), tuple(30003, 5600), tuple(41103, 6145), tuple(83490, 81076), tuple(55059, 66736), tuple(45015, 10239), tuple(79555, 85819), tuple(81808, 34970), tuple(19235, 85480), tuple(91807, 52177), tuple(40887, 87009), tuple(5003, 2687), tuple(64964, 88122), tuple(765, 94893), tuple(93573, 20504), tuple(28854, 38438), tuple(94244, 93475), tuple(72996, 84801), tuple(75427, 81692), tuple(63161, 98637), tuple(18814, 61343), tuple(22863, 60110), tuple(8949, 12694), tuple(19675, 94313), tuple(43857, 74073), tuple(15737, 58218), tuple(48895, 68474), tuple(22220, 92926), tuple(69055, 50282), tuple(40532, 74934), tuple(59062, 66405), tuple(85784, 87704), tuple(58494, 88222), tuple(2260, 20401), tuple(73112, 99666), tuple(46739, 95433), tuple(21179, 85119), tuple(11545, 38801), tuple(59993, 50866), tuple(10086, 4709), tuple(70560, 29611), tuple(27095, 89017), tuple(6896, 2279), tuple(92506, 5013), tuple(48600, 90491), tuple(18782, 54638), tuple(54337, 82734), tuple(52054, 13481), tuple(38297, 56559), tuple(15998, 30591), tuple(89789, 7522), tuple(18149, 28725), tuple(3532, 28625), tuple(70934, 49617), tuple(84599, 55664), tuple(74229, 52269), tuple(55431, 11893), tuple(32807, 72543), tuple(83882, 53025), tuple(11490, 83442), tuple(14844, 88612), tuple(12526, 45953), tuple(906, 2231), tuple(68240, 95612), tuple(18818, 31535), tuple(57774, 91290), tuple(67250, 67400), tuple(77332, 23550), tuple(42332, 57775), tuple(28792, 11539), tuple(19108, 34608), tuple(12399, 38591), tuple(7329, 10740), tuple(84288, 50928), tuple(29461, 17629), tuple(63884, 88489), tuple(47479, 61085), tuple(75357, 57255), tuple(60107, 94046), tuple(32934, 66312), tuple(28615, 42600), tuple(55553, 85213), tuple(57838, 91426), tuple(9783, 11513), tuple(73677, 28821), tuple(75408, 75561), tuple(22995, 59224), tuple(74874, 54145), tuple(18513, 75901), tuple(46440, 69414), tuple(36072, 22263), tuple(60560, 73325), tuple(69967, 93358), tuple(75949, 98634), tuple(3688, 57991), tuple(43482, 94541), tuple(40922, 31011), tuple(57763, 74497), tuple(93576, 96392), tuple(83038, 80656), tuple(47757, 87045), tuple(14061, 53465), tuple(65619, 33775), tuple(11341, 6702), tuple(6249, 87358), tuple(15766, 85937), tuple(13135, 93945), tuple(24495, 95900), tuple(80359, 1739), tuple(15468, 73426), tuple(49240, 44999), tuple(82839, 90808), tuple(87438, 75613), tuple(348, 73144), tuple(99523, 85853), tuple(21557, 70210), tuple(64933, 1672), tuple(38154, 17477), tuple(97136, 67363), tuple(96491, 8038), tuple(97981, 3434), tuple(54372, 27038), tuple(88480, 86675), tuple(21028, 21083), tuple(43197, 4440), tuple(31702, 78290), tuple(66631, 24438), tuple(11482, 17922), tuple(90351, 39503), tuple(46186, 32439), tuple(73828, 6640), tuple(56916, 26029), tuple(62840, 1815), tuple(20281, 28488), tuple(18211, 30043), tuple(65211, 93012), tuple(43614, 58012), tuple(90322, 77343), tuple(64293, 94525), tuple(59489, 39760), tuple(93219, 78440), tuple(74613, 9732), tuple(38085, 19191), tuple(58029, 48186), tuple(88762, 1764), tuple(28627, 21993), tuple(49975, 41225), tuple(70486, 43480), tuple(82764, 96425), tuple(27218, 78327), tuple(17844, 73333), tuple(70463, 37629), tuple(10500, 33826), tuple(97343, 66575), tuple(82833, 51210), tuple(77353, 45073), tuple(27163, 39728), tuple(78076, 46691), tuple(80302, 39342), tuple(77142, 1319), tuple(87403, 80110), tuple(53805, 27786), tuple(50558, 74264), tuple(83146, 31358), tuple(11567, 4438), tuple(30041, 54287), tuple(91731, 18496), tuple(57591, 93894), tuple(72534, 59009), tuple(98064, 59148), tuple(69626, 66615), tuple(20951, 43949), tuple(61960, 68060), tuple(48892, 67918), tuple(61321, 56222), tuple(75424, 77260), tuple(4916, 81929), tuple(68892, 81531), tuple(28096, 28548), tuple(62016, 107), tuple(8593, 12030), tuple(66743, 36772), tuple(60174, 15106), tuple(52844, 1923), tuple(34768, 22065), tuple(88988, 62910), tuple(79214, 2998), tuple(25675, 31376), tuple(69959, 3614), tuple(43885, 31708), tuple(12206, 46548), tuple(69924, 19343), tuple(12984, 38980), tuple(58250, 69438), tuple(2580, 48684), tuple(38112, 37124), tuple(21842, 43150), tuple(59384, 21921), tuple(19908, 46678), tuple(73396, 79529), tuple(8274, 1557), tuple(36975, 65519), tuple(81069, 18712), tuple(13692, 9148), tuple(60617, 84762), tuple(75749, 66154), tuple(80375, 24553), tuple(4257, 47056), tuple(76880, 7687), tuple(40714, 43448), tuple(79112, 74791), tuple(33119, 72730), tuple(17670, 89183), tuple(51614, 3921), tuple(21247, 39857), tuple(86756, 67673), tuple(32792, 70035), tuple(5917, 7197), tuple(1762, 23130), tuple(6455, 63664), tuple(32806, 3729), tuple(60469, 20511), tuple(12522, 15149), tuple(98106, 79338), tuple(84754, 11162), tuple(52058, 17973), tuple(28789, 1521), tuple(32766, 36325), tuple(78914, 40453), tuple(70297, 71854), tuple(9313, 45190), tuple(54559, 66227), tuple(22342, 43860), tuple(44152, 84294), tuple(36913, 93173), tuple(88523, 36338), tuple(82234, 71140), tuple(8328, 22947), tuple(73250, 88125), tuple(74356, 16820), tuple(94472, 37349), tuple(23126, 87806), tuple(40315, 88729), tuple(19935, 19145), tuple(93312, 65719), tuple(8477, 33108), tuple(86660, 69525), tuple(75557, 66964), tuple(60437, 57494), tuple(94419, 42524), tuple(95372, 72274), tuple(49866, 85685), tuple(96808, 39404), tuple(62961, 72507), tuple(38634, 70815), tuple(91379, 42430), tuple(66359, 98699), tuple(24382, 4186), tuple(4003, 77760), tuple(87840, 75265), tuple(57641, 68871), tuple(9773, 15942), tuple(5664, 51289), tuple(47923, 31308), tuple(58632, 82468), tuple(14097, 71829), tuple(1838, 97710), tuple(70433, 11364), tuple(82363, 97879), tuple(25257, 20615), tuple(18249, 6758), tuple(98581, 13639), tuple(3290, 72449), tuple(74546, 79380), tuple(97254, 44448), tuple(80316, 31760), tuple(40516, 94809), tuple(14444, 88981), tuple(9693, 10259), tuple(83795, 95485), tuple(70201, 81014), tuple(66644, 16761), tuple(35529, 82718), tuple(75774, 73476), tuple(80139, 3957), tuple(34803, 80689), tuple(92085, 46499), tuple(97871, 8004), tuple(67369, 11354), tuple(43578, 81596), tuple(94695, 44963), tuple(93741, 41629), tuple(16005, 96652), tuple(37918, 69012), tuple(89832, 56041), tuple(51798, 32386), tuple(89749, 27647), tuple(76279, 7990), tuple(31746, 1346), tuple(40841, 20480), tuple(99942, 24473), tuple(78495, 99194), tuple(13588, 57088), tuple(22183, 42297), tuple(82707, 34435), tuple(45026, 12747), tuple(8000, 93211), tuple(40453, 13025), tuple(44100, 39880), tuple(83900, 56474), tuple(87691, 42802), tuple(82000, 63867), tuple(76627, 84731), tuple(112, 92774), tuple(34749, 97737), tuple(59262, 57169), tuple(95571, 44144), tuple(74310, 68970), tuple(63232, 92744), tuple(53698, 21610), tuple(39969, 75475), tuple(39942, 28713), tuple(81230, 50140), tuple(97953, 96528), tuple(86144, 48041), tuple(96677, 49677), tuple(22051, 48183), tuple(33436, 54784), tuple(5553, 11920), tuple(67057, 17115), tuple(57736, 72309), tuple(8086, 85329), tuple(72623, 94949), tuple(13096, 48992), tuple(63153, 56337), tuple(30462, 1036), tuple(75000, 24048), tuple(62635, 50128), tuple(91480, 83131), tuple(25929, 79809), tuple(96237, 76974), tuple(59645, 20603), tuple(31850, 29754), tuple(91070, 36568), tuple(30191, 33785), tuple(86776, 67259), tuple(49073, 39179), tuple(16121, 73834), tuple(84217, 52951), tuple(95866, 47244), tuple(63326, 73460), tuple(134, 91953), tuple(48189, 86069), tuple(42971, 3700), tuple(28643, 10479), tuple(80151, 7446), tuple(78798, 2655), tuple(39135, 69364), tuple(80244, 24904), tuple(22818, 74964), tuple(26753, 82419), tuple(16858, 5212), tuple(79891, 11215), tuple(785, 46103), tuple(12559, 24617), tuple(73601, 71490), tuple(70342, 7099), tuple(73330, 6665), tuple(11903, 28194), tuple(16375, 37746), tuple(86132, 51788), tuple(90345, 68366), tuple(5464, 78338), tuple(23569, 83141), tuple(17904, 94046), tuple(35868, 60017), tuple(22591, 93373), tuple(70584, 72116), tuple(49331, 34312), tuple(16180, 91286), tuple(58494, 65441), tuple(9336, 52671), tuple(32523, 26734), tuple(40205, 83549), tuple(28810, 96876), tuple(44703, 38944), tuple(46981, 37157), tuple(8582, 7529), tuple(59718, 71700), tuple(62545, 73716), tuple(6531, 23200), tuple(30528, 59720), tuple(57152, 84660), tuple(16232, 67946), tuple(60446, 45983), tuple(68737, 54959), tuple(57795, 73107), tuple(26930, 35938), tuple(9844, 44760), tuple(3716, 79020), tuple(99126, 8264), tuple(66120, 16151), tuple(50616, 25765), tuple(93340, 95875), tuple(34103, 88003), tuple(14879, 99758), tuple(49188, 6087), tuple(89858, 42861), tuple(36730, 72076), tuple(25069, 26403), tuple(98183, 48108), tuple(3229, 5367), tuple(59306, 80078), tuple(61144, 58598), tuple(72600, 98765), tuple(57701, 23177), tuple(10176, 11553), tuple(82964, 13697), tuple(7788, 28538), tuple(39943, 97491), tuple(56261, 17781), tuple(2458, 1892), tuple(6679, 45554), tuple(42171, 66222), tuple(24420, 44115), tuple(35852, 41965), tuple(50196, 49555), tuple(34718, 60734), tuple(6932, 61638), tuple(69472, 56723), tuple(489, 97620), tuple(41335, 90578), tuple(1333, 92787), tuple(97883, 64754), tuple(14208, 22097), tuple(75776, 5938), tuple(67446, 61518), tuple(58743, 45162), tuple(34749, 81243), tuple(71451, 91991), tuple(27804, 41836), tuple(45274, 8039), tuple(17593, 24498), tuple(8801, 38559), tuple(87460, 7109), tuple(50075, 18284), tuple(84043, 82146), tuple(62932, 25018), tuple(89647, 56768), tuple(59920, 80801), tuple(56357, 35142), tuple(97376, 58181), tuple(70715, 91103), tuple(90829, 78985), tuple(29776, 13275), tuple(30546, 42320), tuple(99266, 35340), tuple(21234, 61062), tuple(39239, 10745), tuple(45990, 715), tuple(47047, 6619), tuple(4270, 94575), tuple(90009, 72203), tuple(25629, 2691), tuple(67926, 89112), tuple(46990, 61101), tuple(22355, 69536), tuple(1977, 56723), tuple(54681, 34041), tuple(83819, 7024), tuple(81235, 7093), tuple(16659, 87135), tuple(49384, 32135), tuple(42204, 17362), tuple(90585, 70374), tuple(51255, 1), tuple(31600, 70085), tuple(90189, 95778), tuple(57349, 87789), tuple(83384, 93771), tuple(20718, 15529), tuple(10644, 53591), tuple(84103, 62101), tuple(91340, 48382), tuple(82854, 84420), tuple(12561, 53517), tuple(64835, 45362), tuple(54154, 75841), tuple(46498, 31175), tuple(75035, 49552), tuple(9306, 53403), tuple(68851, 49139), tuple(13463, 42107), tuple(2322, 36695), tuple(55953, 12098), tuple(60656, 80482), tuple(78987, 19632), tuple(31228, 18523), tuple(98972, 80489), tuple(32367, 98405), tuple(25139, 5164), tuple(5692, 60610), tuple(36535, 70097), tuple(80542, 74320), tuple(87984, 46750), tuple(98201, 41341), tuple(35217, 46813), tuple(81795, 69057), tuple(83927, 41032), tuple(60149, 26087), tuple(39954, 48361), tuple(64485, 61448), tuple(87185, 14580), tuple(74559, 93251), tuple(88544, 83366), tuple(74015, 15864), tuple(78623, 69719), tuple(16941, 80710), tuple(16315, 58313), tuple(47277, 59107), tuple(16067, 66290), tuple(63906, 59891), tuple(20754, 67817), tuple(44428, 10652), tuple(95960, 99045), tuple(52163, 26221), tuple(65566, 22057), tuple(26836, 38898), tuple(57107, 79274), tuple(39020, 74857), tuple(53540, 84159), tuple(76646, 44324), tuple(27967, 40171), tuple(28710, 56332), tuple(84036, 28711), tuple(68742, 57241), tuple(40535, 34737), tuple(62681, 85386), tuple(30472, 58405), tuple(85086, 33013), tuple(67059, 47481), tuple(30441, 55098), tuple(97892, 71991), tuple(90296, 42905), tuple(22441, 18863), tuple(19606, 77242), tuple(11206, 58380), tuple(23901, 49962), tuple(84094, 33761), tuple(64400, 28093), tuple(64228, 94543), tuple(71874, 20871), tuple(25385, 73117), tuple(63398, 20999), tuple(77547, 51893), tuple(80783, 65858), tuple(39807, 80754), tuple(10336, 90318), tuple(7826, 55346), tuple(30206, 10711), tuple(94411, 67364), tuple(33509, 14329), tuple(65350, 17006), tuple(65999, 55699), tuple(82753, 61081), tuple(38851, 11896), tuple(15155, 48635), tuple(19985, 75204), tuple(37144, 5344), tuple(26173, 39587), tuple(61111, 30966), tuple(16180, 22987), tuple(60707, 43599), tuple(30136, 74118), tuple(7880, 43857), tuple(97445, 30233), tuple(62700, 24828), tuple(90914, 89452), tuple(64131, 56925), tuple(25259, 39132), tuple(47104, 43950), tuple(93891, 21913), tuple(84573, 91029), tuple(8604, 79858), tuple(33141, 25534), tuple(12468, 90413), tuple(97063, 76359), tuple(80826, 26061), tuple(64013, 99099), tuple(82158, 38882), tuple(25799, 7564), tuple(25477, 69847), tuple(73374, 58520), tuple(48230, 9453), tuple(91424, 72273), tuple(64893, 11750), tuple(46753, 48434), tuple(15974, 94633), tuple(14872, 27027), tuple(14527, 21313), tuple(25660, 64644), tuple(54196, 15138), tuple(6313, 10911), tuple(36168, 47170), tuple(45346, 76), tuple(10305, 60286), tuple(65283, 39977), tuple(21804, 37972), tuple(65389, 86954), tuple(90674, 64458), tuple(15838, 22392), tuple(43540, 42503), tuple(49584, 67828), tuple(56711, 87887), tuple(40075, 73696), tuple(23832, 91552), tuple(39002, 65562), tuple(20451, 64664), tuple(70783, 92171), tuple(29319, 57694), tuple(56217, 44247), tuple(52856, 57873), tuple(80560, 90902), tuple(31068, 11280), tuple(46996, 34739), tuple(57527, 4554), tuple(8410, 25816), tuple(12269, 38319), tuple(88054, 49939), tuple(337, 13231), tuple(56432, 68236), tuple(74841, 21476), tuple(96006, 15712), tuple(87145, 91660), tuple(58090, 55111), tuple(10310, 79789), tuple(5734, 79710), tuple(98992, 69026), tuple(77033, 5734), tuple(43338, 42635), tuple(23898, 28669), tuple(62708, 81652), tuple(41279, 51722), tuple(93444, 26355), tuple(62046, 52199), tuple(71492, 58736), tuple(7379, 62581), tuple(8592, 71885), tuple(75026, 40387), tuple(46696, 3939), tuple(9787, 88907), tuple(86356, 363), tuple(97479, 20358), tuple(77363, 65553), tuple(44036, 22178), tuple(98279, 64612), tuple(3615, 411), tuple(77003, 93018), tuple(41605, 88489), tuple(55992, 83614), tuple(19493, 21633), tuple(34639, 97064), tuple(94602, 89289), tuple(45853, 26299), tuple(84170, 73386), tuple(9221, 51439), tuple(41513, 68166), tuple(37170, 17690), tuple(82511, 59246), tuple(96674, 27574), tuple(99301, 45675), tuple(42716, 41520), tuple(56623, 49130), tuple(84100, 76804), tuple(73855, 97007), tuple(73303, 26912), tuple(37151, 23837), tuple(49190, 97104), tuple(23487, 45628), tuple(87763, 46550), tuple(65111, 92605), tuple(80481, 8151), tuple(83949, 18930), tuple(81749, 27244), tuple(37449, 3023), tuple(28303, 51545), tuple(96441, 93242), tuple(22082, 43254), tuple(35135, 68407), tuple(37712, 48709), tuple(5111, 26774), tuple(15532, 74246), tuple(93605, 83583), tuple(21491, 66472), tuple(38922, 53076), tuple(55455, 54432), tuple(955, 44063), tuple(311, 91630), tuple(53554, 4522), tuple(29927, 65668), tuple(7525, 16035), tuple(44093, 76745), tuple(21481, 78198), tuple(76875, 5306), tuple(56126, 76437), tuple(96534, 16880), tuple(85600, 68336), tuple(4479, 81002), tuple(80414, 11593), tuple(8186, 61147), tuple(5624, 32879), tuple(79312, 20995), tuple(40407, 41512), tuple(91261, 66022), tuple(93228, 75364), tuple(21136, 40111), tuple(92148, 60681), tuple(42549, 7944), tuple(60157, 15040), tuple(63562, 88365), tuple(69056, 72713), tuple(78263, 89223), tuple(3776, 33039), tuple(30042, 59984), tuple(64567, 20977), tuple(24720, 39157), tuple(63582, 75653), tuple(45363, 20249), tuple(58093, 53833), tuple(27918, 93306), tuple(25791, 92686), tuple(15904, 862), tuple(72093, 19257), tuple(64125, 88986), tuple(41717, 27989), tuple(43165, 98675), tuple(76840, 48170), tuple(64508, 3535), tuple(91964, 33435), tuple(96686, 88673), tuple(66648, 64594), tuple(17927, 30539), tuple(73615, 22800), tuple(18580, 48077), tuple(59803, 48202), From 708f280605211d93754c6fd498c974ed6bf799a1 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Sat, 15 May 2021 09:35:00 +0300 Subject: [PATCH 06/40] fix the test --- tests/performance/rewrite_sumIf.xml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/performance/rewrite_sumIf.xml b/tests/performance/rewrite_sumIf.xml index 4ba4916bdbf..cc25b28adb7 100644 --- a/tests/performance/rewrite_sumIf.xml +++ b/tests/performance/rewrite_sumIf.xml @@ -1,4 +1,5 @@ - SELECT sumIf(1, 0) FROM numbers(100000000) - SELECT sumIf(1, 1) FROM numbers(100000000) + + SELECT sumIf(1, 0) FROM numbers(1000000000) + SELECT sumIf(1, 1) FROM numbers(1000000000) From 7719c1d933893aaf58380558fdb7e097a5913e36 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Sat, 15 May 2021 09:43:56 +0300 Subject: [PATCH 07/40] collect profiles during warmup --- docker/test/performance-comparison/perf.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 77ffde795b3..cb769be2b12 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -274,7 +274,14 @@ for query_index in queries_to_run: try: # Will also detect too long queries during warmup stage - res = c.execute(q, query_id = prewarm_id, settings = {'max_execution_time': args.max_query_seconds}) + # During the warmup runs, we will also: + # * detect queries that are exceedingly long, to fail fast, + # * collect profiler traces, which might be helpful for analyzing + # test coverage. We disable profiler for normal runs because + # it makes the results unstable. + res = c.execute(q, query_id = prewarm_id, + settings = {'max_execution_time': args.max_query_seconds, + 'query_profiler_real_time_period_ns': 10000000}) except clickhouse_driver.errors.Error as e: # Add query id to the exception to make debugging easier. e.args = (prewarm_id, *e.args) From 91835369cf04b7e1e90a158d92d7c2639dc37244 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Sat, 15 May 2021 09:44:14 +0300 Subject: [PATCH 08/40] fixup --- docker/test/performance-comparison/perf.py | 1 - 1 file changed, 1 deletion(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index cb769be2b12..00b8eb686ef 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -273,7 +273,6 @@ for query_index in queries_to_run: prewarm_id = f'{query_prefix}.prewarm0' try: - # Will also detect too long queries during warmup stage # During the warmup runs, we will also: # * detect queries that are exceedingly long, to fail fast, # * collect profiler traces, which might be helpful for analyzing From 6afd9f84f1b0511bf7bb5e1b01bf9eb84fcbd8e0 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Wed, 19 May 2021 13:25:43 +0300 Subject: [PATCH 09/40] longer --- docker/test/performance-comparison/perf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/test/performance-comparison/perf.py b/docker/test/performance-comparison/perf.py index 00b8eb686ef..9628c512e83 100755 --- a/docker/test/performance-comparison/perf.py +++ b/docker/test/performance-comparison/perf.py @@ -44,7 +44,7 @@ parser.add_argument('--port', nargs='*', default=[9000], help="Space-separated l parser.add_argument('--runs', type=int, default=1, help='Number of query runs per server.') parser.add_argument('--max-queries', type=int, default=None, help='Test no more than this number of queries, chosen at random.') parser.add_argument('--queries-to-run', nargs='*', type=int, default=None, help='Space-separated list of indexes of queries to test.') -parser.add_argument('--max-query-seconds', type=int, default=10, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.') +parser.add_argument('--max-query-seconds', type=int, default=15, help='For how many seconds at most a query is allowed to run. The script finishes with error if this time is exceeded.') parser.add_argument('--profile-seconds', type=int, default=0, help='For how many seconds to profile a query for which the performance has changed.') parser.add_argument('--long', action='store_true', help='Do not skip the tests tagged as long.') parser.add_argument('--print-queries', action='store_true', help='Print test queries and exit.') From a1749cf5c3a341f536af028cd23a9323bf7aaa73 Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 20 May 2021 15:05:20 +0300 Subject: [PATCH 10/40] fix --- docker/test/performance-comparison/report.py | 2 +- tests/performance/agg_functions_min_max_any.xml | 2 +- tests/performance/set_index.xml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docker/test/performance-comparison/report.py b/docker/test/performance-comparison/report.py index 94765664f57..503f2904461 100755 --- a/docker/test/performance-comparison/report.py +++ b/docker/test/performance-comparison/report.py @@ -560,7 +560,7 @@ if args.report == 'main': # treat as errors. if very_unstable_queries: status = 'failure' - message_array.append(str(unstable_queries) + ' unstable') + message_array.append(str(very_unstable_queries) + ' unstable') error_tests += slow_average_tests if error_tests: diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index 6ca9e3eb65a..4b6ce59fa7c 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/set_index.xml b/tests/performance/set_index.xml index 1fb7cf967f3..4d5cc5a56ed 100644 --- a/tests/performance/set_index.xml +++ b/tests/performance/set_index.xml @@ -1,4 +1,4 @@ - + CREATE TABLE test_in (`a` UInt32) ENGINE = MergeTree() ORDER BY a INSERT INTO test_in SELECT number FROM numbers(500000000) From aeb84310045ab2c2c2fae24eb61a7a88835820de Mon Sep 17 00:00:00 2001 From: Alexander Kuzmenkov Date: Thu, 20 May 2021 17:14:55 +0300 Subject: [PATCH 11/40] update thresholds - 1 --- tests/performance/agg_functions_min_max_any.xml | 2 +- tests/performance/array_join.xml | 2 +- tests/performance/avg_weighted.xml | 2 +- tests/performance/bounding_ratio.xml | 2 +- tests/performance/codecs_float_insert.xml | 3 +-- tests/performance/codecs_int_insert.xml | 2 +- tests/performance/collations.xml | 2 +- tests/performance/date_time_64.xml | 2 +- tests/performance/date_time_long.xml | 2 +- tests/performance/decimal_aggregates.xml | 2 +- tests/performance/decimal_casts.xml | 2 +- tests/performance/direct_dictionary.xml | 2 +- tests/performance/flat_dictionary.xml | 2 +- tests/performance/float_parsing.xml | 2 +- tests/performance/fuzz_bits.xml | 2 +- tests/performance/general_purpose_hashes_on_UUID.xml | 2 +- tests/performance/generate_table_function.xml | 2 +- tests/performance/group_by_sundy_li.xml | 2 +- tests/performance/hashed_dictionary.xml | 2 +- tests/performance/if_array_string.xml | 2 +- tests/performance/int_parsing.xml | 2 +- tests/performance/jit_small_requests.xml | 2 +- tests/performance/joins_in_memory_pmj.xml | 2 +- tests/performance/json_extract_rapidjson.xml | 2 +- tests/performance/json_extract_simdjson.xml | 2 +- tests/performance/merge_tree_many_partitions_2.xml | 2 +- tests/performance/optimized_select_final.xml | 2 +- tests/performance/parallel_mv.xml | 2 +- tests/performance/point_in_polygon.xml | 2 +- tests/performance/polymorphic_parts_l.xml | 2 +- tests/performance/polymorphic_parts_m.xml | 2 +- tests/performance/polymorphic_parts_s.xml | 2 +- tests/performance/questdb_sum_int32.xml | 2 +- tests/performance/random_string.xml | 2 +- tests/performance/redundant_functions_in_order_by.xml | 2 +- tests/performance/set_index.xml | 2 +- tests/performance/sum_map.xml | 2 +- tests/performance/synthetic_hardware_benchmark.xml | 2 +- tests/performance/uniq.xml | 2 +- tests/performance/url_hits.xml | 2 +- tests/performance/visit_param_extract_raw.xml | 2 +- tests/performance/website.xml | 2 +- 42 files changed, 42 insertions(+), 43 deletions(-) diff --git a/tests/performance/agg_functions_min_max_any.xml b/tests/performance/agg_functions_min_max_any.xml index 4b6ce59fa7c..f609e6ddecc 100644 --- a/tests/performance/agg_functions_min_max_any.xml +++ b/tests/performance/agg_functions_min_max_any.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/array_join.xml b/tests/performance/array_join.xml index cf92b51f545..ca280ce28ad 100644 --- a/tests/performance/array_join.xml +++ b/tests/performance/array_join.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/avg_weighted.xml b/tests/performance/avg_weighted.xml index 2476011e6a9..fa39dd552cd 100644 --- a/tests/performance/avg_weighted.xml +++ b/tests/performance/avg_weighted.xml @@ -1,4 +1,4 @@ - + 1 1 diff --git a/tests/performance/bounding_ratio.xml b/tests/performance/bounding_ratio.xml index e430136b624..e3a15f90013 100644 --- a/tests/performance/bounding_ratio.xml +++ b/tests/performance/bounding_ratio.xml @@ -1,4 +1,4 @@ - + SELECT boundingRatio(number, number) FROM numbers(100000000) SELECT (argMax(number, number) - argMin(number, number)) / (max(number) - min(number)) FROM numbers(100000000) diff --git a/tests/performance/codecs_float_insert.xml b/tests/performance/codecs_float_insert.xml index b282bcc268f..b31e0eafdd7 100644 --- a/tests/performance/codecs_float_insert.xml +++ b/tests/performance/codecs_float_insert.xml @@ -1,5 +1,4 @@ - - + 1 diff --git a/tests/performance/codecs_int_insert.xml b/tests/performance/codecs_int_insert.xml index 662df80ae70..caefaba3725 100644 --- a/tests/performance/codecs_int_insert.xml +++ b/tests/performance/codecs_int_insert.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/collations.xml b/tests/performance/collations.xml index 52ccede3798..17b2d36b7e3 100644 --- a/tests/performance/collations.xml +++ b/tests/performance/collations.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/date_time_64.xml b/tests/performance/date_time_64.xml index fd883416a33..838aba34d87 100644 --- a/tests/performance/date_time_64.xml +++ b/tests/performance/date_time_64.xml @@ -1,4 +1,4 @@ - + hits_100m_single diff --git a/tests/performance/date_time_long.xml b/tests/performance/date_time_long.xml index c2eb42d3318..0c3d85f9659 100644 --- a/tests/performance/date_time_long.xml +++ b/tests/performance/date_time_long.xml @@ -1,4 +1,4 @@ - + datetime_transform diff --git a/tests/performance/decimal_aggregates.xml b/tests/performance/decimal_aggregates.xml index 3fc1408d7e4..7078cb16002 100644 --- a/tests/performance/decimal_aggregates.xml +++ b/tests/performance/decimal_aggregates.xml @@ -1,4 +1,4 @@ - + 35G diff --git a/tests/performance/decimal_casts.xml b/tests/performance/decimal_casts.xml index 6dd38b6a06a..f087d0938c1 100644 --- a/tests/performance/decimal_casts.xml +++ b/tests/performance/decimal_casts.xml @@ -1,4 +1,4 @@ - + 15G diff --git a/tests/performance/direct_dictionary.xml b/tests/performance/direct_dictionary.xml index 3f01449ed99..7983c058928 100644 --- a/tests/performance/direct_dictionary.xml +++ b/tests/performance/direct_dictionary.xml @@ -1,4 +1,4 @@ - + CREATE TABLE simple_key_direct_dictionary_source_table ( diff --git a/tests/performance/flat_dictionary.xml b/tests/performance/flat_dictionary.xml index a80631db541..243f90a8e59 100644 --- a/tests/performance/flat_dictionary.xml +++ b/tests/performance/flat_dictionary.xml @@ -1,4 +1,4 @@ - + CREATE TABLE simple_key_flat_dictionary_source_table ( diff --git a/tests/performance/float_parsing.xml b/tests/performance/float_parsing.xml index eb8577bd127..33ab8ba6f10 100644 --- a/tests/performance/float_parsing.xml +++ b/tests/performance/float_parsing.xml @@ -1,4 +1,4 @@ - + expr diff --git a/tests/performance/fuzz_bits.xml b/tests/performance/fuzz_bits.xml index 87064e520c2..2679977cb1d 100644 --- a/tests/performance/fuzz_bits.xml +++ b/tests/performance/fuzz_bits.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/general_purpose_hashes_on_UUID.xml b/tests/performance/general_purpose_hashes_on_UUID.xml index 3694e0a38bd..ccff0619c6e 100644 --- a/tests/performance/general_purpose_hashes_on_UUID.xml +++ b/tests/performance/general_purpose_hashes_on_UUID.xml @@ -1,4 +1,4 @@ - + hash_func diff --git a/tests/performance/generate_table_function.xml b/tests/performance/generate_table_function.xml index 0339a8c19e8..bc49a7de1bd 100644 --- a/tests/performance/generate_table_function.xml +++ b/tests/performance/generate_table_function.xml @@ -1,4 +1,4 @@ - + SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8') LIMIT 1000000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('ui64 UInt64, i64 Int64, ui32 UInt32, i32 Int32, ui16 UInt16, i16 Int16, ui8 UInt8, i8 Int8', 0, 10, 10) LIMIT 1000000000); SELECT sum(NOT ignore(*)) FROM (SELECT * FROM generateRandom('i Enum8(\'hello\' = 1, \'world\' = 5)', 0, 10, 10) LIMIT 1000000000); diff --git a/tests/performance/group_by_sundy_li.xml b/tests/performance/group_by_sundy_li.xml index aebc305335c..1728d5b3253 100644 --- a/tests/performance/group_by_sundy_li.xml +++ b/tests/performance/group_by_sundy_li.xml @@ -1,4 +1,4 @@ - + 8 diff --git a/tests/performance/hashed_dictionary.xml b/tests/performance/hashed_dictionary.xml index 26164b4f888..8dfb33ca53d 100644 --- a/tests/performance/hashed_dictionary.xml +++ b/tests/performance/hashed_dictionary.xml @@ -1,4 +1,4 @@ - + CREATE TABLE simple_key_hashed_dictionary_source_table ( diff --git a/tests/performance/if_array_string.xml b/tests/performance/if_array_string.xml index f1752767e76..e4bf4fe8412 100644 --- a/tests/performance/if_array_string.xml +++ b/tests/performance/if_array_string.xml @@ -1,4 +1,4 @@ - + SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : ['a', 'b', 'c']) SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? materialize(['Hello', 'World']) : ['a', 'b', 'c']) SELECT count() FROM zeros(10000000) WHERE NOT ignore(rand() % 2 ? ['Hello', 'World'] : materialize(['a', 'b', 'c'])) diff --git a/tests/performance/int_parsing.xml b/tests/performance/int_parsing.xml index 32f904331ce..3b8620e46c3 100644 --- a/tests/performance/int_parsing.xml +++ b/tests/performance/int_parsing.xml @@ -1,4 +1,4 @@ - + hits_100m_single hits_10m_single diff --git a/tests/performance/jit_small_requests.xml b/tests/performance/jit_small_requests.xml index d8f917fb9af..c9abec0926b 100644 --- a/tests/performance/jit_small_requests.xml +++ b/tests/performance/jit_small_requests.xml @@ -1,4 +1,4 @@ - + WITH bitXor(number, 0x4CF2D2BAAE6DA887) AS x0, diff --git a/tests/performance/joins_in_memory_pmj.xml b/tests/performance/joins_in_memory_pmj.xml index 05d1876cb06..d122dba72c3 100644 --- a/tests/performance/joins_in_memory_pmj.xml +++ b/tests/performance/joins_in_memory_pmj.xml @@ -1,4 +1,4 @@ - + CREATE TABLE ints (i64 Int64, i32 Int32, i16 Int16, i8 Int8) ENGINE = Memory diff --git a/tests/performance/json_extract_rapidjson.xml b/tests/performance/json_extract_rapidjson.xml index 2a34acfe10f..f8d40c1e58d 100644 --- a/tests/performance/json_extract_rapidjson.xml +++ b/tests/performance/json_extract_rapidjson.xml @@ -1,4 +1,4 @@ - + diff --git a/tests/performance/json_extract_simdjson.xml b/tests/performance/json_extract_simdjson.xml index 9ec3613d5e8..87b1eaac73e 100644 --- a/tests/performance/json_extract_simdjson.xml +++ b/tests/performance/json_extract_simdjson.xml @@ -1,4 +1,4 @@ - + json diff --git a/tests/performance/merge_tree_many_partitions_2.xml b/tests/performance/merge_tree_many_partitions_2.xml index 0b8a4650835..6799153ed65 100644 --- a/tests/performance/merge_tree_many_partitions_2.xml +++ b/tests/performance/merge_tree_many_partitions_2.xml @@ -1,4 +1,4 @@ - + CREATE TABLE bad_partitions (a UInt64, b UInt64, c UInt64, d UInt64, e UInt64, f UInt64, g UInt64, h UInt64, i UInt64, j UInt64, k UInt64, l UInt64, m UInt64, n UInt64, o UInt64, p UInt64, q UInt64, r UInt64, s UInt64, t UInt64, u UInt64, v UInt64, w UInt64, x UInt64, y UInt64, z UInt64) ENGINE = MergeTree PARTITION BY x ORDER BY x INSERT INTO bad_partitions (x) SELECT * FROM numbers_mt(3000) diff --git a/tests/performance/optimized_select_final.xml b/tests/performance/optimized_select_final.xml index d70fccc1330..46b408d0cb4 100644 --- a/tests/performance/optimized_select_final.xml +++ b/tests/performance/optimized_select_final.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/parallel_mv.xml b/tests/performance/parallel_mv.xml index e6635a74b96..1e9b92c54df 100644 --- a/tests/performance/parallel_mv.xml +++ b/tests/performance/parallel_mv.xml @@ -1,4 +1,4 @@ - + 1 diff --git a/tests/performance/point_in_polygon.xml b/tests/performance/point_in_polygon.xml index 31c24eb006f..a22affb8e63 100644 --- a/tests/performance/point_in_polygon.xml +++ b/tests/performance/point_in_polygon.xml @@ -1,4 +1,4 @@ - +