Merge pull request #58349 from azat/ci/flake8

Check python code with flake8

docker/test/style/Dockerfile

@@ -30,6 +30,7 @@ RUN pip3 install \
         mypy==1.8.0 \
         pylint==3.1.0 \
         python-magic==0.4.24 \
+        flake8==4.0.1 \
         requests \
         thefuzz \
         types-requests \

docker/test/style/run.sh

@@ -9,6 +9,8 @@ echo "Check style" | ts
 ./check-style -n              |& tee /test_output/style_output.txt
 echo "Check python formatting with black" | ts
 ./check-black -n              |& tee /test_output/black_output.txt
+echo "Check python with flake8" | ts
+./check-flake8                |& tee /test_output/flake8_output.txt
 echo "Check python type hinting with mypy" | ts
 ./check-mypy -n               |& tee /test_output/mypy_output.txt
 echo "Check typos" | ts

docs/en/development/continuous-integration.md

@@ -91,6 +91,9 @@ cd ./utils/check-style
 # Check python type hinting with mypy
 ./check-mypy
 
+# Check python with flake8
+./check-flake8
+
 # Check code with codespell
 ./check-typos
 

tests/integration/helpers/hdfs_api.py

@@ -110,10 +110,9 @@ class HDFSApi(object):
                         logging.debug(
                             "Stdout:\n{}\n".format(res.stdout.decode("utf-8"))
                         )
-                        logging.debug("Env:\n{}\n".format(env))
                         raise Exception(
                             "Command {} return non-zero code {}: {}".format(
-                                args, res.returncode, res.stderr.decode("utf-8")
+                                cmd, res.returncode, res.stderr.decode("utf-8")
                             )
                         )
 

tests/integration/helpers/uclient.py

@@ -8,7 +8,7 @@ sys.path.insert(0, os.path.join(CURDIR))
 
 from . import uexpect
 
-prompt = ":\) "
+prompt = ":\\) "
 end_of_block = r".*\r\n.*\r\n"
 
 
@@ -21,7 +21,7 @@ class client(object):
         self.client.eol("\r")
         self.client.logger(log, prefix=name)
         self.client.timeout(20)
-        self.client.expect("[#\$] ", timeout=2)
+        self.client.expect("[#\\$] ", timeout=2)
         self.client.send(command)
 
     def __enter__(self):

tests/integration/test_backup_restore_new/test.py

@@ -1474,7 +1474,7 @@ def test_backup_all(exclude_system_log_tables):
     restore_settings = []
     if not exclude_system_log_tables:
         restore_settings.append("allow_non_empty_tables=true")
-    restore_command = f"RESTORE ALL FROM {backup_name} {'SETTINGS '+ ', '.join(restore_settings) if restore_settings else ''}"
+    restore_command = f"RESTORE ALL FROM {backup_name} {'SETTINGS ' + ', '.join(restore_settings) if restore_settings else ''}"
 
     session_id = new_session_id()
     instance.http_query(

tests/integration/test_backup_restore_on_cluster/test_disallow_concurrency.py

@@ -161,13 +161,13 @@ def wait_for_fail_restore(node, restore_id):
     elif status == "RESTORING":
         assert_eq_with_retry(
             node,
-            f"SELECT status FROM system.backups WHERE id = '{backup_id}'",
+            f"SELECT status FROM system.backups WHERE id = '{restore_id}'",
             "RESTORE_FAILED",
             sleep_time=2,
             retry_count=50,
         )
         error = node.query(
-            f"SELECT error FROM system.backups WHERE id == '{backup_id}'"
+            f"SELECT error FROM system.backups WHERE id == '{restore_id}'"
         ).rstrip("\n")
         assert re.search(
             "Cannot restore the table default.tbl because it already contains some data",

tests/integration/test_backward_compatibility/test_convert_ordinary.py

@@ -187,7 +187,7 @@ def check_convert_all_dbs_to_atomic():
 
     # 6 tables, MVs contain 2 rows (inner tables does not match regexp)
     assert "8\t{}\n".format(8 * len("atomic")) == node.query(
-        "SELECT count(), sum(n) FROM atomic.merge".format(db)
+        "SELECT count(), sum(n) FROM atomic.merge"
     )
 
     node.query("DETACH TABLE ordinary.detached PERMANENTLY")

tests/integration/test_backward_compatibility/test_functions.py

@@ -89,7 +89,7 @@ def test_aggregate_states(start_cluster):
                 logging.info("Skipping %s", aggregate_function)
                 skipped += 1
                 continue
-            logging.exception("Failed %s", function)
+            logging.exception("Failed %s", aggregate_function)
             failed += 1
             continue
 

tests/integration/test_disk_over_web_server/test.py

@@ -116,7 +116,7 @@ def test_usage(cluster, node_name):
             (id Int32) ENGINE = MergeTree() ORDER BY id
             SETTINGS storage_policy = 'web';
         """.format(
-                i, uuids[i], i, i
+                i, uuids[i]
             )
         )
 
@@ -338,7 +338,7 @@ def test_page_cache(cluster):
             (id Int32) ENGINE = MergeTree() ORDER BY id
             SETTINGS storage_policy = 'web';
         """.format(
-                i, uuids[i], i, i
+                i, uuids[i]
             )
         )
 

tests/integration/test_jbod_balancer/test.py

@@ -90,7 +90,7 @@ def wait_until_fully_merged(node, table):
         except:
             return
 
-    raise Exception(f"There are still merges on-going after {retry} assignments")
+    raise Exception(f"There are still merges on-going after {i} assignments")
 
 
 def test_jbod_balanced_merge(start_cluster):

tests/integration/test_jdbc_bridge/test.py

@@ -91,7 +91,7 @@ def test_jdbc_insert(started_cluster):
         """
         CREATE TABLE test.test_insert ENGINE = Memory AS
         SELECT * FROM test.ClickHouseTable;
-        SELECT * 
+        SELECT *
         FROM jdbc('{0}?mutation', 'INSERT INTO test.test_insert VALUES({1}, ''{1}'', ''{1}'')');
     """.format(
             datasource, records
@@ -115,7 +115,7 @@ def test_jdbc_update(started_cluster):
         """
         CREATE TABLE test.test_update ENGINE = Memory AS
         SELECT * FROM test.ClickHouseTable;
-        SELECT * 
+        SELECT *
         FROM jdbc(
             '{}?mutation',
             'SET mutations_sync = 1; ALTER TABLE test.test_update UPDATE Str=''{}'' WHERE Num = {} - 1;'
@@ -145,7 +145,7 @@ def test_jdbc_delete(started_cluster):
         """
         CREATE TABLE test.test_delete ENGINE = Memory AS
         SELECT * FROM test.ClickHouseTable;
-        SELECT * 
+        SELECT *
         FROM jdbc(
             '{}?mutation',
             'SET mutations_sync = 1; ALTER TABLE test.test_delete DELETE WHERE Num < {} - 1;'
@@ -158,7 +158,7 @@ def test_jdbc_delete(started_cluster):
     expected = records - 1
     actual = instance.query(
         "SELECT Str FROM jdbc('{}', 'SELECT * FROM test.test_delete')".format(
-            datasource, records
+            datasource
         )
     )
     assert int(actual) == expected, "expecting {} but got {}".format(expected, actual)

tests/integration/test_keeper_snapshot_small_distance/test.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-##!/usr/bin/env python3
+
 import pytest
 from helpers.cluster import ClickHouseCluster
 import helpers.keeper_utils as keeper_utils

tests/integration/test_keeper_snapshots/test.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-#!/usr/bin/env python3
 import pytest
 from helpers.cluster import ClickHouseCluster
 import helpers.keeper_utils as keeper_utils

tests/integration/test_keeper_three_nodes_start/test.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python3
 
-#!/usr/bin/env python3
 import pytest
 from helpers.cluster import ClickHouseCluster
 import random

tests/integration/test_merge_tree_azure_blob_storage/test.py

@@ -537,10 +537,7 @@ def test_freeze_unfreeze(cluster):
 def test_apply_new_settings(cluster):
     node = cluster.instances[NODE_NAME]
     create_table(node, TABLE_NAME)
-    config_path = os.path.join(
-        SCRIPT_DIR,
-        "./_gen/disk_storage_conf.xml".format(cluster.instances_dir_name),
-    )
+    config_path = os.path.join(SCRIPT_DIR, "./_gen/disk_storage_conf.xml")
 
     azure_query(
         node, f"INSERT INTO {TABLE_NAME} VALUES {generate_values('2020-01-03', 4096)}"

tests/integration/test_postgresql_replica_database_engine_1/test.py

@@ -179,9 +179,7 @@ def test_different_data_types(started_cluster):
     for i in range(10):
         col = random.choice(["a", "b", "c"])
         cursor.execute("UPDATE test_data_types SET {} = {};".format(col, i))
-        cursor.execute(
-            """UPDATE test_data_types SET i = '2020-12-12';""".format(col, i)
-        )
+        cursor.execute("UPDATE test_data_types SET i = '2020-12-12';")
 
     check_tables_are_synchronized(instance, "test_data_types", "id")
 
@@ -452,7 +450,7 @@ def test_many_concurrent_queries(started_cluster):
                     # also change primary key value
                     print("try update primary key {}".format(thread_id))
                     cursor.execute(
-                        "UPDATE {table}_{} SET key=key%100000+100000*{} WHERE key%{}=0".format(
+                        "UPDATE {} SET key=key%100000+100000*{} WHERE key%{}=0".format(
                             table_name, i + 1, i + 1
                         )
                     )

tests/integration/test_prometheus_endpoint/test.py

@@ -28,7 +28,7 @@ def parse_response_line(line):
 
     if line.startswith("#"):
         return {}
-    match = re.match("^([a-zA-Z_:][a-zA-Z0-9_:]+)(\{.*\})? -?(\d)", line)
+    match = re.match(r"^([a-zA-Z_:][a-zA-Z0-9_:]+)(\{.*\})? -?(\d)", line)
     assert match, line
     name, _, val = match.groups()
     return {name: int(val)}

tests/integration/test_scheduler/test.py

@@ -6,6 +6,7 @@ import time
 import threading
 import pytest
 
+from helpers.client import QueryRuntimeException
 from helpers.cluster import ClickHouseCluster
 
 cluster = ClickHouseCluster(__file__)

tests/integration/test_storage_hudi/test.py

@@ -4,7 +4,7 @@ import os
 import json
 
 import helpers.client
-from helpers.cluster import ClickHouseCluster
+from helpers.cluster import ClickHouseCluster, ClickHouseInstance
 from helpers.test_tools import TSV
 from helpers.s3_tools import prepare_s3_bucket, upload_directory, get_file_contents
 

tests/integration/test_storage_iceberg/test.py

@@ -1,5 +1,5 @@
 import helpers.client
-from helpers.cluster import ClickHouseCluster
+from helpers.cluster import ClickHouseCluster, ClickHouseInstance
 from helpers.test_tools import TSV
 
 import pyspark

tests/integration/test_storage_rabbitmq/test.py

@@ -702,7 +702,7 @@ def test_rabbitmq_sharding_between_queues_publish(rabbitmq_cluster):
 
     assert (
         int(result1) == messages_num * threads_num
-    ), "ClickHouse lost some messages: {}".format(result)
+    ), "ClickHouse lost some messages: {}".format(result1)
     assert int(result2) == 10
 
 
@@ -1516,7 +1516,7 @@ def test_rabbitmq_hash_exchange(rabbitmq_cluster):
 
     assert (
         int(result1) == messages_num * threads_num
-    ), "ClickHouse lost some messages: {}".format(result)
+    ), "ClickHouse lost some messages: {}".format(result1)
     assert int(result2) == 4 * num_tables
 
 
@@ -1966,7 +1966,7 @@ def test_rabbitmq_many_consumers_to_each_queue(rabbitmq_cluster):
 
     assert (
         int(result1) == messages_num * threads_num
-    ), "ClickHouse lost some messages: {}".format(result)
+    ), "ClickHouse lost some messages: {}".format(result1)
     # 4 tables, 2 consumers for each table => 8 consumer tags
     assert int(result2) == 8
 
@@ -2427,9 +2427,7 @@ def test_rabbitmq_drop_table_properly(rabbitmq_cluster):
     time.sleep(30)
 
     try:
-        exists = channel.queue_declare(
-            callback, queue="rabbit_queue_drop", passive=True
-        )
+        exists = channel.queue_declare(queue="rabbit_queue_drop", passive=True)
     except Exception as e:
         exists = False
 
@@ -3364,7 +3362,7 @@ def test_rabbitmq_flush_by_block_size(rabbitmq_cluster):
                     routing_key="",
                     body=json.dumps({"key": 0, "value": 0}),
                 )
-            except e:
+            except Exception as e:
                 logging.debug(f"Got error: {str(e)}")
 
     produce_thread = threading.Thread(target=produce)
@@ -3442,7 +3440,7 @@ def test_rabbitmq_flush_by_time(rabbitmq_cluster):
                 )
                 logging.debug("Produced a message")
                 time.sleep(0.8)
-            except e:
+            except Exception as e:
                 logging.debug(f"Got error: {str(e)}")
 
     produce_thread = threading.Thread(target=produce)

tests/integration/test_ttl_move/test.py

@@ -1850,7 +1850,7 @@ class TestCancelBackgroundMoving:
         config = inspect.cleandoc(
             f"""
             <clickhouse>
-                <max_local_write_bandwidth_for_server>{ 256 * 1024 }</max_local_write_bandwidth_for_server>
+                <max_local_write_bandwidth_for_server>{256 * 1024}</max_local_write_bandwidth_for_server>
             </clickhouse>
             """
         )

tests/integration/test_ttl_replicated/test.py

@@ -325,7 +325,7 @@ def optimize_with_retry(node, table_name, retry=20):
                 settings={"optimize_throw_if_noop": "1"},
             )
             break
-        except e:
+        except:
             time.sleep(0.5)
 
 

tests/queries/0_stateless/01056_window_view_proc_hop_watch.py

@@ -49,16 +49,16 @@ with client(name="client1>", log=log) as client1, client(
 
     client1.send("WATCH 01056_window_view_proc_hop_watch.wv")
     client1.expect("Query id" + end_of_block)
-    client1.expect("Progress: 0.00 rows.*\)")
+    client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01056_window_view_proc_hop_watch.mt VALUES (1, now('US/Samoa') + 3)"
     )
     client1.expect("1" + end_of_block)
-    client1.expect("Progress: 1.00 rows.*\)")
+    client1.expect("Progress: 1.00 rows.*\\)")
 
     # send Ctrl-C
     client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
+    match = client1.expect("(%s)|([#\\$] )" % prompt)
     if match.groups()[1]:
         client1.send(client1.command)
         client1.expect(prompt)

tests/queries/0_stateless/01059_window_view_event_hop_watch_strict_asc.py

@@ -47,7 +47,7 @@ with client(name="client1>", log=log) as client1, client(
 
     client1.send("WATCH db_01059_event_hop_watch_strict_asc.wv")
     client1.expect("Query id" + end_of_block)
-    client1.expect("Progress: 0.00 rows.*\)")
+    client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO db_01059_event_hop_watch_strict_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));"
     )
@@ -57,7 +57,7 @@ with client(name="client1>", log=log) as client1, client(
     )
     client2.expect("Ok.")
     client1.expect("1*1990-01-01 12:00:02" + end_of_block)
-    client1.expect("Progress: 1.00 rows.*\)")
+    client1.expect("Progress: 1.00 rows.*\\)")
 
     client2.send(
         "INSERT INTO db_01059_event_hop_watch_strict_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:10', 'US/Samoa'));"
@@ -65,11 +65,11 @@ with client(name="client1>", log=log) as client1, client(
     client2.expect("Ok.")
     client1.expect("1*1990-01-01 12:00:06" + end_of_block)
     client1.expect("1*1990-01-01 12:00:08" + end_of_block)
-    client1.expect("Progress: 3.00 rows.*\)")
+    client1.expect("Progress: 3.00 rows.*\\)")
 
     # send Ctrl-C
     client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
+    match = client1.expect("(%s)|([#\\$] )" % prompt)
     if match.groups()[1]:
         client1.send(client1.command)
         client1.expect(prompt)

tests/queries/0_stateless/01062_window_view_event_hop_watch_asc.py

@@ -49,7 +49,7 @@ with client(name="client1>", log=log) as client1, client(
 
     client1.send("WATCH 01062_window_view_event_hop_watch_asc.wv")
     client1.expect("Query id" + end_of_block)
-    client1.expect("Progress: 0.00 rows.*\)")
+    client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01062_window_view_event_hop_watch_asc.mt VALUES (1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));"
     )
@@ -69,11 +69,11 @@ with client(name="client1>", log=log) as client1, client(
     client2.expect(prompt)
     client1.expect("1" + end_of_block)
     client1.expect("2" + end_of_block)
-    client1.expect("Progress: 3.00 rows.*\)")
+    client1.expect("Progress: 3.00 rows.*\\)")
 
     # send Ctrl-C
     client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
+    match = client1.expect("(%s)|([#\\$] )" % prompt)
     if match.groups()[1]:
         client1.send(client1.command)
         client1.expect(prompt)

tests/queries/0_stateless/01065_window_view_event_hop_watch_bounded.py

@@ -50,7 +50,7 @@ with client(name="client1>", log=log) as client1, client(
 
     client1.send("WATCH 01065_window_view_event_hop_watch_bounded.wv")
     client1.expect("Query id" + end_of_block)
-    client1.expect("Progress: 0.00 rows.*\)")
+    client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01065_window_view_event_hop_watch_bounded.mt VALUES (1, '1990/01/01 12:00:00');"
     )
@@ -72,7 +72,7 @@ with client(name="client1>", log=log) as client1, client(
 
     # send Ctrl-C
     client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
+    match = client1.expect("(%s)|([#\\$] )" % prompt)
     if match.groups()[1]:
         client1.send(client1.command)
         client1.expect(prompt)

tests/queries/0_stateless/01069_window_view_proc_tumble_watch.py

@@ -49,23 +49,23 @@ with client(name="client1>", log=log) as client1, client(
 
     client1.send("WATCH 01069_window_view_proc_tumble_watch.wv")
     client1.expect("Query id" + end_of_block)
-    client1.expect("Progress: 0.00 rows.*\)")
+    client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01069_window_view_proc_tumble_watch.mt VALUES (1, now('US/Samoa') + 3)"
     )
     client2.expect("Ok.")
     client1.expect("1" + end_of_block)
-    client1.expect("Progress: 1.00 rows.*\)")
+    client1.expect("Progress: 1.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01069_window_view_proc_tumble_watch.mt VALUES (1, now('US/Samoa') + 3)"
     )
     client2.expect("Ok.")
     client1.expect("1" + end_of_block)
-    client1.expect("Progress: 2.00 rows.*\)")
+    client1.expect("Progress: 2.00 rows.*\\)")
 
     # send Ctrl-C
     client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
+    match = client1.expect("(%s)|([#\\$] )" % prompt)
     if match.groups()[1]:
         client1.send(client1.command)
         client1.expect(prompt)

tests/queries/0_stateless/01070_window_view_watch_events.py

@@ -49,7 +49,7 @@ with client(name="client1>", log=log) as client1, client(
 
     client1.send("WATCH 01070_window_view_watch_events.wv EVENTS")
     client1.expect("Query id" + end_of_block)
-    client1.expect("Progress: 0.00 rows.*\)")
+    client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01070_window_view_watch_events.mt VALUES (1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));"
     )
@@ -59,11 +59,11 @@ with client(name="client1>", log=log) as client1, client(
     )
     client2.expect("Ok.")
     client1.expect("1990-01-01 12:00:05" + end_of_block)
-    client1.expect("Progress: 1.00 rows.*\)")
+    client1.expect("Progress: 1.00 rows.*\\)")
 
     # send Ctrl-C
     client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
+    match = client1.expect("(%s)|([#\\$] )" % prompt)
     if match.groups()[1]:
         client1.send(client1.command)
         client1.expect(prompt)

tests/queries/0_stateless/01078_window_view_alter_query_watch.py

@@ -55,7 +55,7 @@ with client(name="client1>", log=log) as client1, client(
 
     client1.send("WATCH 01078_window_view_alter_query_watch.wv")
     client1.expect("Query id" + end_of_block)
-    client1.expect("Progress: 0.00 rows.*\)")
+    client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01078_window_view_alter_query_watch.mt VALUES (1, toDateTime('1990/01/01 12:00:00', 'US/Samoa'));"
     )
@@ -65,7 +65,7 @@ with client(name="client1>", log=log) as client1, client(
     )
     client2.expect("Ok.")
     client1.expect("1" + end_of_block)
-    client1.expect("Progress: 1.00 rows.*\)")
+    client1.expect("Progress: 1.00 rows.*\\)")
     client2.send(
         "ALTER TABLE 01078_window_view_alter_query_watch.wv MODIFY QUERY SELECT count(a) * 2 AS count, hopEnd(wid) AS w_end FROM 01078_window_view_alter_query_watch.mt GROUP BY hop(timestamp, INTERVAL '2' SECOND, INTERVAL '3' SECOND, 'US/Samoa') AS wid"
     )
@@ -75,7 +75,7 @@ with client(name="client1>", log=log) as client1, client(
     client1.expect(prompt)
     client3.send("WATCH 01078_window_view_alter_query_watch.wv")
     client3.expect("Query id" + end_of_block)
-    client3.expect("Progress: 0.00 rows.*\)")
+    client3.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01078_window_view_alter_query_watch.mt VALUES (1, toDateTime('1990/01/01 12:00:06', 'US/Samoa'));"
     )
@@ -85,11 +85,11 @@ with client(name="client1>", log=log) as client1, client(
     )
     client2.expect("Ok.")
     client3.expect("2" + end_of_block)
-    client3.expect("Progress: 1.00 rows.*\)")
+    client3.expect("Progress: 1.00 rows.*\\)")
 
     # send Ctrl-C
     client3.send("\x03", eol="")
-    match = client3.expect("(%s)|([#\$] )" % prompt)
+    match = client3.expect("(%s)|([#\\$] )" % prompt)
     if match.groups()[1]:
         client3.send(client3.command)
         client3.expect(prompt)

tests/queries/0_stateless/01082_window_view_watch_limit.py

@@ -49,7 +49,7 @@ with client(name="client1>", log=log) as client1, client(
 
     client1.send("WATCH 01082_window_view_watch_limit.wv LIMIT 1")
     client1.expect("Query id" + end_of_block)
-    client1.expect("Progress: 0.00 rows.*\)")
+    client1.expect("Progress: 0.00 rows.*\\)")
     client2.send(
         "INSERT INTO 01082_window_view_watch_limit.mt VALUES (1, '1990/01/01 12:00:00');"
     )
@@ -59,7 +59,7 @@ with client(name="client1>", log=log) as client1, client(
     )
     client2.expect("Ok.")
     client1.expect("1" + end_of_block)
-    client1.expect("Progress: 1.00 rows.*\)")
+    client1.expect("Progress: 1.00 rows.*\\)")
     client1.expect("1 row" + end_of_block)
     client1.expect(prompt)
 

tests/queries/0_stateless/01921_test_progress_bar.py

@@ -15,6 +15,6 @@ log = None
 with client(name="client1>", log=log) as client1:
     client1.expect(prompt)
     client1.send("SELECT number FROM numbers(1000) FORMAT Null")
-    client1.expect("Progress: 1\.00 thousand rows, 8\.00 KB .*" + end_of_block)
-    client1.expect("0 rows in set. Elapsed: [\\w]{1}\.[\\w]{3} sec.")
+    client1.expect("Progress: 1\\.00 thousand rows, 8\\.00 KB .*" + end_of_block)
+    client1.expect("0 rows in set. Elapsed: [\\w]{1}\\.[\\w]{3} sec.")
     client1.expect("Peak memory usage: .*B" + end_of_block)

tests/queries/0_stateless/02473_infile_progress.py

@@ -32,12 +32,12 @@ with client(
     )
     client1.expect(prompt)
     client1.send(f"INSERT INTO test.infile_progress FROM INFILE '{filename}'")
-    client1.expect("Progress: 5.00 rows, 10.00 B.*\)")
+    client1.expect("Progress: 5.00 rows, 10.00 B.*\\)")
     client1.expect(prompt)
 
     # send Ctrl-C
     client1.send("\x03", eol="")
-    match = client1.expect("(%s)|([#\$] )" % prompt)
+    match = client1.expect("(%s)|([#\\$] )" % prompt)
     if match.groups()[1]:
         client1.send(client1.command)
         client1.expect(prompt)

tests/queries/0_stateless/helpers/client.py

@@ -8,7 +8,7 @@ sys.path.insert(0, os.path.join(CURDIR))
 
 import uexpect
 
-prompt = ":\) "
+prompt = ":\\) "
 end_of_block = r".*\r\n.*\r\n"
 
 
@@ -21,7 +21,7 @@ class client(object):
         self.client.eol("\r")
         self.client.logger(log, prefix=name)
         self.client.timeout(120)
-        self.client.expect("[#\$] ", timeout=60)
+        self.client.expect("[#\\$] ", timeout=60)
         self.client.send(command)
 
     def __enter__(self):

tests/queries/0_stateless/helpers/shell.py

@@ -10,7 +10,7 @@ import uexpect
 
 
 class shell(object):
-    def __init__(self, command=None, name="", log=None, prompt="[#\$] "):
+    def __init__(self, command=None, name="", log=None, prompt="[#\\$] "):
         if command is None:
             command = ["/bin/bash", "--noediting"]
         self.prompt = prompt

utils/backupview/clickhouse_backupview.py

@@ -8,6 +8,7 @@ import shutil
 
 import zipfile  # For reading backups from zip archives
 import boto3  # For reading backups from S3
+import botocore
 
 
 ## Examples:

utils/check-style/check-flake8

@@ -0,0 +1,55 @@
+#!/usr/bin/env bash
+
+function join_by() { local IFS="$1"; shift; echo "$*"; }
+
+set -e
+
+# We check only our code, that's why we skip contrib
+GIT_ROOT=$(git rev-parse --show-cdup)
+GIT_ROOT=${GIT_ROOT:-./}
+
+# Find all *.py, *.python files and executable files without extension
+# that are determined as python scripts by 'file' util
+# in the repo except the contrib directory.
+find_cmd=(
+  find "$GIT_ROOT" -type f -not -path "${GIT_ROOT}contrib/*"
+    \(
+      \(
+        -name '*.py' -or -name "*.python" -or
+          \(
+            -executable -not -name "*.*" -exec sh -c 'file {} | grep -q "Python script"' \;
+          \)
+      \)
+      # We skip modules generated by the protocol buffer compiler from *.proto files.
+      -and -not -name '*_pb2.py' -and -not -name '*_pb2_grpc.py'
+    \) -print0
+)
+
+ignores=(
+    E101 # Indentation contains mixed spaces and tabs
+    E203 # Whitespace before ':'
+    E226 # missing whitespace around arithmetic operator
+    E266 # Too many leading '#' for block comment
+    E401 # Multiple imports on one line
+    E402 # Module level import not at top of file
+    E501 # line too long
+    E711 # Comparison to None should be 'cond is None:'
+    E712 # Comparison to true should be 'if cond is true:' or 'if cond:'
+    E713 # Test for membership should be 'not in'
+    E714 # Test for object identity should be 'is not'
+    E722 # Do not use bare except, specify exception instead
+    E731 # Do not assign a lambda expression, use a def
+    E741 # Do not use variables named 'I', 'O', or 'l'
+    F401 # Module imported but unused
+    F403 # 'from module import *' used; unable to detect undefined names
+    F405 # Name may be undefined, or defined from star imports: module
+    F522 # .format(...) unused named arguments
+    F541 # f-string without any placeholders
+    F811 # redefinition of unused name from line N
+    F841 # local variable name is assigned to but never used
+    W191 # Indentation contains tabs
+    W291 # Trailing whitespace
+    W293 # Blank line contains whitespace
+    W503 # Line break occurred before a binary operator
+)
+"${find_cmd[@]}" | xargs -0 flake8 --ignore "$(join_by , "${ignores[@]}")"

utils/check-style/process_style_check_result.py

@@ -18,6 +18,7 @@ def process_result(result_folder):
         "style",
         "pylint",
         "black",
+        "flake8",
         "mypy",
         "typos",
         "whitespaces",