Merge branch 'master' into Issue_39395

.github/workflows/master.yml

@@ -151,8 +151,8 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  SplitBuildSmokeTest:
-    needs: [BuilderDebSplitted]
+  SharedBuildSmokeTest:
+    needs: [BuilderDebShared]
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Set envs
@@ -171,7 +171,7 @@ jobs:
         uses: actions/download-artifact@v2
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Split build check
+      - name: Shared build check
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
@@ -598,7 +598,7 @@ jobs:
 ##########################################################################################
 ##################################### SPECIAL BUILDS #####################################
 ##########################################################################################
-  BuilderDebSplitted:
+  BuilderDebShared:
     needs: [DockerHubPush]
     runs-on: [self-hosted, builder]
     steps:
@@ -609,7 +609,7 @@ jobs:
           IMAGES_PATH=${{runner.temp}}/images_path
           REPO_COPY=${{runner.temp}}/build_check/ClickHouse
           CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=binary_splitted
+          BUILD_NAME=binary_shared
           EOF
       - name: Download changed images
         uses: actions/download-artifact@v2
@@ -1012,7 +1012,7 @@ jobs:
       # - BuilderBinGCC
       - BuilderBinPPC64
       - BuilderBinClangTidy
-      - BuilderDebSplitted
+      - BuilderDebShared
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Set envs
@@ -3153,7 +3153,7 @@ jobs:
       - UnitTestsMsan
       - UnitTestsUBsan
       - UnitTestsReleaseClang
-      - SplitBuildSmokeTest
+      - SharedBuildSmokeTest
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Clear repository

.github/workflows/pull_request.yml

@@ -216,8 +216,8 @@ jobs:
           # shellcheck disable=SC2046
           docker rm -f $(docker ps -a -q) ||:
           sudo rm -fr "$TEMP_PATH"
-  SplitBuildSmokeTest:
-    needs: [BuilderDebSplitted]
+  SharedBuildSmokeTest:
+    needs: [BuilderDebShared]
     runs-on: [self-hosted, style-checker]
     steps:
       - name: Set envs
@@ -236,7 +236,7 @@ jobs:
         uses: actions/download-artifact@v2
         with:
           path: ${{ env.REPORTS_PATH }}
-      - name: Split build check
+      - name: Shared build check
         run: |
           sudo rm -fr "$TEMP_PATH"
           mkdir -p "$TEMP_PATH"
@@ -620,7 +620,7 @@ jobs:
 ##########################################################################################
 ##################################### SPECIAL BUILDS #####################################
 ##########################################################################################
-  BuilderDebSplitted:
+  BuilderDebShared:
     needs: [DockerHubPush, FastTest, StyleCheck]
     runs-on: [self-hosted, builder]
     steps:
@@ -631,7 +631,7 @@ jobs:
           IMAGES_PATH=${{runner.temp}}/images_path
           REPO_COPY=${{runner.temp}}/build_check/ClickHouse
           CACHES_PATH=${{runner.temp}}/../ccaches
-          BUILD_NAME=binary_splitted
+          BUILD_NAME=binary_shared
           EOF
       - name: Download changed images
         uses: actions/download-artifact@v2
@@ -1024,7 +1024,7 @@ jobs:
       # - BuilderBinGCC
       - BuilderBinPPC64
       - BuilderBinClangTidy
-      - BuilderDebSplitted
+      - BuilderDebShared
     runs-on: [self-hosted, style-checker]
     if: ${{ success() || failure() }}
     steps:
@@ -3416,7 +3416,7 @@ jobs:
       - UnitTestsMsan
       - UnitTestsUBsan
       - UnitTestsReleaseClang
-      - SplitBuildSmokeTest
+      - SharedBuildSmokeTest
       - CompatibilityCheck
       - IntegrationTestsFlakyCheck
       - Jepsen

README.md

@@ -15,4 +15,4 @@ ClickHouse® is an open-source column-oriented database management system that a
 * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any.
 
 ## Upcoming events
-* **v22.8 Release Webinar** Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.
+* [**v22.8 Release Webinar**](https://clickhouse.com/company/events/v22-8-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap.

contrib/arrow

@@ -1 +1 @@
-Subproject commit 3e03c6de41a86df2fc54a61e9be1abaefeff6b0e
+Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5

docker/packager/packager

@@ -100,12 +100,12 @@ def run_docker_image_with_env(
     subprocess.check_call(cmd, shell=True)
 
 
-def is_release_build(build_type, package_type, sanitizer, split_binary):
+def is_release_build(build_type, package_type, sanitizer, shared_libraries):
     return (
         build_type == ""
         and package_type == "deb"
         and sanitizer == ""
-        and not split_binary
+        and not shared_libraries
     )
 
 
@@ -116,7 +116,7 @@ def parse_env_variables(
     package_type,
     cache,
     distcc_hosts,
-    split_binary,
+    shared_libraries,
     clang_tidy,
     version,
     author,
@@ -202,7 +202,7 @@ def parse_env_variables(
         cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr")
         cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc")
         cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var")
-        if is_release_build(build_type, package_type, sanitizer, split_binary):
+        if is_release_build(build_type, package_type, sanitizer, shared_libraries):
             cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON")
             result.append("WITH_PERFORMANCE=1")
             if is_cross_arm:
@@ -219,7 +219,7 @@ def parse_env_variables(
     if package_type == "coverity":
         result.append("COMBINED_OUTPUT=coverity")
         result.append('COVERITY_TOKEN="$COVERITY_TOKEN"')
-    elif split_binary:
+    elif shared_libraries:
         result.append("COMBINED_OUTPUT=shared_build")
 
     if sanitizer:
@@ -264,7 +264,7 @@ def parse_env_variables(
         result.append("BINARY_OUTPUT=tests")
         cmake_flags.append("-DENABLE_TESTS=1")
 
-    if split_binary:
+    if shared_libraries:
         cmake_flags.append(
             "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1"
         )
@@ -351,7 +351,7 @@ if __name__ == "__main__":
         default="",
     )
 
-    parser.add_argument("--split-binary", action="store_true")
+    parser.add_argument("--shared-libraries", action="store_true")
     parser.add_argument("--clang-tidy", action="store_true")
     parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="")
     parser.add_argument(
@@ -404,7 +404,7 @@ if __name__ == "__main__":
         args.package_type,
         args.cache,
         args.distcc_hosts,
-        args.split_binary,
+        args.shared_libraries,
         args.clang_tidy,
         args.version,
         args.author,

docs/en/operations/server-configuration-parameters/settings.md

@@ -982,7 +982,7 @@ Default value: 2.
 **Example**
 
 ```xml
-<background_merges_mutations_concurrency_ratio>3</background_pbackground_merges_mutations_concurrency_ratio>
+<background_merges_mutations_concurrency_ratio>3</background_merges_mutations_concurrency_ratio>
 ```
 
 ## background_move_pool_size {#background_move_pool_size}

src/Common/ProfileEvents.cpp

@@ -25,6 +25,10 @@
     M(WriteBufferFromFileDescriptorWrite, "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.") \
     M(WriteBufferFromFileDescriptorWriteFailed, "Number of times the write (write/pwrite) to a file descriptor have failed.") \
     M(WriteBufferFromFileDescriptorWriteBytes, "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.") \
+    M(FileSync, "Number of times the F_FULLFSYNC/fsync/fdatasync function was called for files.") \
+    M(DirectorySync, "Number of times the F_FULLFSYNC/fsync/fdatasync function was called for directories.") \
+    M(FileSyncElapsedMicroseconds, "Total time spent waiting for F_FULLFSYNC/fsync/fdatasync syscall for files.") \
+    M(DirectorySyncElapsedMicroseconds, "Total time spent waiting for F_FULLFSYNC/fsync/fdatasync syscall for directories.") \
     M(ReadCompressedBytes, "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).") \
     M(CompressedReadBufferBlocks, "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).") \
     M(CompressedReadBufferBytes, "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).") \

src/Dictionaries/SSDCacheDictionaryStorage.h

@@ -34,6 +34,8 @@ namespace ProfileEvents
     extern const Event AIOWriteBytes;
     extern const Event AIORead;
     extern const Event AIOReadBytes;
+    extern const Event FileSync;
+    extern const Event FileSyncElapsedMicroseconds;
 }
 
 namespace DB
@@ -544,6 +546,9 @@ public:
                 file_path,
                 std::to_string(bytes_written));
 
+        ProfileEvents::increment(ProfileEvents::FileSync);
+
+        Stopwatch watch;
         #if defined(OS_DARWIN)
         if (::fsync(file.fd) < 0)
             throwFromErrnoWithPath("Cannot fsync " + file_path, file_path, ErrorCodes::CANNOT_FSYNC);
@@ -551,6 +556,7 @@ public:
         if (::fdatasync(file.fd) < 0)
             throwFromErrnoWithPath("Cannot fdatasync " + file_path, file_path, ErrorCodes::CANNOT_FSYNC);
         #endif
+        ProfileEvents::increment(ProfileEvents::FileSyncElapsedMicroseconds, watch.elapsedMicroseconds());
 
         current_block_index += buffer_size_in_blocks;
 

src/Disks/LocalDirectorySyncGuard.cpp

@@ -1,6 +1,8 @@
 #include <Disks/LocalDirectorySyncGuard.h>
+#include <Common/ProfileEvents.h>
 #include <Common/Exception.h>
 #include <Disks/IDisk.h>
+#include <Common/Stopwatch.h>
 #include <fcntl.h> // O_RDWR
 
 /// OSX does not have O_DIRECTORY
@@ -8,6 +10,12 @@
 #define O_DIRECTORY O_RDWR
 #endif
 
+namespace ProfileEvents
+{
+    extern const Event DirectorySync;
+    extern const Event DirectorySyncElapsedMicroseconds;
+}
+
 namespace DB
 {
 
@@ -29,8 +37,12 @@ LocalDirectorySyncGuard::LocalDirectorySyncGuard(const String & full_path)
 
 LocalDirectorySyncGuard::~LocalDirectorySyncGuard()
 {
+    ProfileEvents::increment(ProfileEvents::DirectorySync);
+
     try
     {
+        Stopwatch watch;
+
 #if defined(OS_DARWIN)
         if (fcntl(fd, F_FULLFSYNC, 0))
             throwFromErrno("Cannot fcntl(F_FULLFSYNC)", ErrorCodes::CANNOT_FSYNC);
@@ -40,6 +52,8 @@ LocalDirectorySyncGuard::~LocalDirectorySyncGuard()
 #endif
         if (-1 == ::close(fd))
             throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE);
+
+        ProfileEvents::increment(ProfileEvents::DirectorySyncElapsedMicroseconds, watch.elapsedMicroseconds());
     }
     catch (...)
     {

src/IO/WriteBufferFromFileDescriptor.cpp

@@ -18,6 +18,8 @@ namespace ProfileEvents
     extern const Event WriteBufferFromFileDescriptorWriteFailed;
     extern const Event WriteBufferFromFileDescriptorWriteBytes;
     extern const Event DiskWriteElapsedMicroseconds;
+    extern const Event FileSync;
+    extern const Event FileSyncElapsedMicroseconds;
 }
 
 namespace CurrentMetrics
@@ -113,12 +115,18 @@ void WriteBufferFromFileDescriptor::sync()
     /// If buffer has pending data - write it.
     next();
 
+    ProfileEvents::increment(ProfileEvents::FileSync);
+
+    Stopwatch watch;
+
     /// Request OS to sync data with storage medium.
 #if defined(OS_DARWIN)
     int res = ::fsync(fd);
 #else
     int res = ::fdatasync(fd);
 #endif
+    ProfileEvents::increment(ProfileEvents::FileSyncElapsedMicroseconds, watch.elapsedMicroseconds());
+
     if (-1 == res)
         throwFromErrnoWithPath("Cannot fsync " + getFileName(), getFileName(), ErrorCodes::CANNOT_FSYNC);
 }

tests/ci/build_check.py

@@ -39,7 +39,7 @@ def _can_export_binaries(build_config: BuildConfig) -> bool:
         return False
     if build_config["bundled"] != "bundled":
         return False
-    if build_config["splitted"] == "splitted":
+    if build_config["libraries"] == "shared":
         return False
     if build_config["sanitizer"] != "":
         return True
@@ -68,8 +68,8 @@ def get_packager_cmd(
         cmd += f" --build-type={build_config['build_type']}"
     if build_config["sanitizer"]:
         cmd += f" --sanitizer={build_config['sanitizer']}"
-    if build_config["splitted"] == "splitted":
-        cmd += " --split-binary"
+    if build_config["libraries"] == "shared":
+        cmd += " --shared-libraries"
     if build_config["tidy"] == "enable":
         cmd += " --clang-tidy"
 

tests/ci/build_report_check.py

@@ -37,7 +37,7 @@ class BuildResult:
         build_type,
         sanitizer,
         bundled,
-        splitted,
+        libraries,
         status,
         elapsed_seconds,
         with_coverage,
@@ -46,7 +46,7 @@ class BuildResult:
         self.build_type = build_type
         self.sanitizer = sanitizer
         self.bundled = bundled
-        self.splitted = splitted
+        self.libraries = libraries
         self.status = status
         self.elapsed_seconds = elapsed_seconds
         self.with_coverage = with_coverage
@@ -91,7 +91,7 @@ def get_failed_report(
         build_type="unknown",
         sanitizer="unknown",
         bundled="unknown",
-        splitted="unknown",
+        libraries="unknown",
         status=message,
         elapsed_seconds=0,
         with_coverage=False,
@@ -108,7 +108,7 @@ def process_report(
         build_type=build_config["build_type"],
         sanitizer=build_config["sanitizer"],
         bundled=build_config["bundled"],
-        splitted=build_config["splitted"],
+        libraries=build_config["libraries"],
         status="success" if build_report["status"] else "failure",
         elapsed_seconds=build_report["elapsed_seconds"],
         with_coverage=False,

tests/ci/ci_config.py

@@ -14,7 +14,7 @@ CI_CONFIG = {
             "package_type": "deb",
             "static_binary_name": "amd64",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "additional_pkgs": True,
             "tidy": "disable",
             "with_coverage": False,
@@ -25,7 +25,7 @@ CI_CONFIG = {
             "sanitizer": "",
             "package_type": "coverity",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
             "official": False,
@@ -37,7 +37,7 @@ CI_CONFIG = {
             "package_type": "deb",
             "static_binary_name": "aarch64",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "additional_pkgs": True,
             "tidy": "disable",
             "with_coverage": False,
@@ -48,7 +48,7 @@ CI_CONFIG = {
             "sanitizer": "address",
             "package_type": "deb",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -58,7 +58,7 @@ CI_CONFIG = {
             "sanitizer": "undefined",
             "package_type": "deb",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -68,7 +68,7 @@ CI_CONFIG = {
             "sanitizer": "thread",
             "package_type": "deb",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -78,7 +78,7 @@ CI_CONFIG = {
             "sanitizer": "memory",
             "package_type": "deb",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -88,7 +88,7 @@ CI_CONFIG = {
             "sanitizer": "",
             "package_type": "deb",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -98,7 +98,7 @@ CI_CONFIG = {
             "sanitizer": "",
             "package_type": "binary",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -109,17 +109,17 @@ CI_CONFIG = {
             "package_type": "binary",
             "static_binary_name": "debug-amd64",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "enable",
             "with_coverage": False,
         },
-        "binary_splitted": {
+        "binary_shared": {
             "compiler": "clang-14",
             "build_type": "",
             "sanitizer": "",
             "package_type": "binary",
             "bundled": "bundled",
-            "splitted": "splitted",
+            "libraries": "shared",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -130,7 +130,7 @@ CI_CONFIG = {
             "package_type": "binary",
             "static_binary_name": "macos",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -140,7 +140,7 @@ CI_CONFIG = {
             "sanitizer": "",
             "package_type": "binary",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -151,7 +151,7 @@ CI_CONFIG = {
             "package_type": "binary",
             "static_binary_name": "freebsd",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -162,7 +162,7 @@ CI_CONFIG = {
             "package_type": "binary",
             "static_binary_name": "macos-aarch64",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -173,7 +173,7 @@ CI_CONFIG = {
             "package_type": "binary",
             "static_binary_name": "powerpc64le",
             "bundled": "bundled",
-            "splitted": "unsplitted",
+            "libraries": "static",
             "tidy": "disable",
             "with_coverage": False,
         },
@@ -192,7 +192,7 @@ CI_CONFIG = {
         ],
         "ClickHouse special build check": [
             "binary_tidy",
-            "binary_splitted",
+            "binary_shared",
             "binary_darwin",
             "binary_aarch64",
             "binary_freebsd",
@@ -297,7 +297,7 @@ CI_CONFIG = {
             "required_build": "package_release",
         },
         "Split build smoke test": {
-            "required_build": "binary_splitted",
+            "required_build": "binary_shared",
         },
         "Unit tests (release-clang)": {
             "required_build": "binary_release",

tests/ci/report.py

@@ -290,7 +290,7 @@ tr:hover td {{filter: brightness(95%);}}
 <th>Build type</th>
 <th>Sanitizer</th>
 <th>Bundled</th>
-<th>Splitted</th>
+<th>Libraries</th>
 <th>Status</th>
 <th>Build log</th>
 <th>Build time</th>
@@ -335,7 +335,7 @@ def create_build_html_report(
             row += "<td>{}</td>".format("none")
 
         row += "<td>{}</td>".format(build_result.bundled)
-        row += "<td>{}</td>".format(build_result.splitted)
+        row += "<td>{}</td>".format(build_result.libraries)
 
         if build_result.status:
             style = _get_status_style(build_result.status)

tests/integration/parallel_skip.json

@@ -46,5 +46,19 @@
   "test_storage_s3/test.py::test_url_reconnect_in_the_middle",
   "test_system_metrics/test.py::test_readonly_metrics",
   "test_system_replicated_fetches/test.py::test_system_replicated_fetches",
-  "test_zookeeper_config_load_balancing/test.py::test_round_robin"
+  "test_zookeeper_config_load_balancing/test.py::test_round_robin",
+
+  "test_tlsv1_3/test.py::test_https",
+  "test_tlsv1_3/test.py::test_https_wrong_cert",
+  "test_tlsv1_3/test.py::test_https_non_ssl_auth",
+  "test_tlsv1_3/test.py::test_create_user",
+  "test_user_ip_restrictions/test.py::test_ipv4",
+  "test_user_ip_restrictions/test.py::test_ipv6",
+  "test_ssl_cert_authentication/test.py::test_https",
+  "test_ssl_cert_authentication/test.py::test_https_wrong_cert",
+  "test_ssl_cert_authentication/test.py::test_https_non_ssl_auth",
+  "test_ssl_cert_authentication/test.py::test_create_user",
+  "test_grpc_protocol_ssl/test.py::test_secure_channel",
+  "test_grpc_protocol_ssl/test.py::test_insecure_channel",
+  "test_grpc_protocol_ssl/test.py::test_wrong_client_certificate"
 ]

tests/integration/test_grpc_protocol_ssl/test.py

@@ -5,7 +5,8 @@ import grpc
 from helpers.cluster import ClickHouseCluster, run_and_check
 
 GRPC_PORT = 9100
-NODE_IP = "10.5.172.77"  # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
+# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
+NODE_IP = "10.5.172.77"  # Never copy-paste this line
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 DEFAULT_ENCODING = "utf-8"
 

tests/integration/test_ssl_cert_authentication/test.py

@@ -5,7 +5,8 @@ import ssl
 import os.path
 
 HTTPS_PORT = 8443
-NODE_IP = "10.5.172.77"  # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
+# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
+NODE_IP = "10.5.172.77"  # Never copy-paste this line
 NODE_IP_WITH_HTTPS_PORT = NODE_IP + ":" + str(HTTPS_PORT)
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 

tests/integration/test_tlsv1_3/test.py

@@ -5,7 +5,8 @@ import ssl
 import os.path
 
 HTTPS_PORT = 8443
-NODE_IP = "10.5.172.77"  # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
+# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf).
+NODE_IP = "10.5.172.77"  # Never copy-paste this line
 NODE_IP_WITH_HTTPS_PORT = NODE_IP + ":" + str(HTTPS_PORT)
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 

tests/integration/test_user_ip_restrictions/test.py

@@ -8,47 +8,56 @@ node_ipv4 = cluster.add_instance(
     "node_ipv4",
     main_configs=[],
     user_configs=["configs/users_ipv4.xml"],
-    ipv4_address="10.5.172.77",
+    ipv4_address="10.5.172.77",  # Never copy-paste this line
 )
 client_ipv4_ok = cluster.add_instance(
-    "client_ipv4_ok", main_configs=[], user_configs=[], ipv4_address="10.5.172.10"
+    "client_ipv4_ok",
+    main_configs=[],
+    user_configs=[],
+    ipv4_address="10.5.172.10",  # Never copy-paste this line
 )
 client_ipv4_ok_direct = cluster.add_instance(
-    "client_ipv4_ok_direct", main_configs=[], user_configs=[], ipv4_address="10.5.173.1"
+    "client_ipv4_ok_direct",
+    main_configs=[],
+    user_configs=[],
+    ipv4_address="10.5.173.1",  # Never copy-paste this line
 )
 client_ipv4_ok_full_mask = cluster.add_instance(
     "client_ipv4_ok_full_mask",
     main_configs=[],
     user_configs=[],
-    ipv4_address="10.5.175.77",
+    ipv4_address="10.5.175.77",  # Never copy-paste this line
 )
 client_ipv4_bad = cluster.add_instance(
-    "client_ipv4_bad", main_configs=[], user_configs=[], ipv4_address="10.5.173.10"
+    "client_ipv4_bad",
+    main_configs=[],
+    user_configs=[],
+    ipv4_address="10.5.173.10",  # Never copy-paste this line
 )
 
 node_ipv6 = cluster.add_instance(
     "node_ipv6",
     main_configs=["configs/config_ipv6.xml"],
     user_configs=["configs/users_ipv6.xml"],
-    ipv6_address="2001:3984:3989::1:1000",
+    ipv6_address="2001:3984:3989::1:1000",  # Never copy-paste this line
 )
 client_ipv6_ok = cluster.add_instance(
     "client_ipv6_ok",
     main_configs=[],
     user_configs=[],
-    ipv6_address="2001:3984:3989::5555",
+    ipv6_address="2001:3984:3989::5555",  # Never copy-paste this line
 )
 client_ipv6_ok_direct = cluster.add_instance(
     "client_ipv6_ok_direct",
     main_configs=[],
     user_configs=[],
-    ipv6_address="2001:3984:3989::1:1111",
+    ipv6_address="2001:3984:3989::1:1111",  # Never copy-paste this line
 )
 client_ipv6_bad = cluster.add_instance(
     "client_ipv6_bad",
     main_configs=[],
     user_configs=[],
-    ipv6_address="2001:3984:3989::1:1112",
+    ipv6_address="2001:3984:3989::1:1112",  # Never copy-paste this line
 )
 
 

tests/queries/0_stateless/02317_distinct_in_order_optimization.reference

@@ -85,11 +85,11 @@ select distinct 1 as x, 2 as y from distinct_in_order order by x;
 1	2
 select distinct 1 as x, 2 as y from distinct_in_order order by x, y;
 1	2
-select distinct a, 1 as x from distinct_in_order order by x;
+select a, x from (select distinct a, 1 as x from distinct_in_order order by x) order by a;
 0	1
 select distinct a, 1 as x, 2 as y from distinct_in_order order by a;
 0	1	2
-select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a;
+select a, b, x, y from(select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a) order by a, b;
 0	0	1	2
 0	1	1	2
 0	2	1	2
@@ -97,10 +97,10 @@ select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a;
 0	4	1	2
 select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by y;
 1	2
-select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by b;
+select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by a, b;
 0	0	1	2
 0	1	1	2
 0	2	1	2
 0	3	1	2
 0	4	1	2
--- check that distinct in order has the same result as ordinary distinct
+-- check that distinct in order returns the same result as ordinary distinct

tests/queries/0_stateless/02317_distinct_in_order_optimization.sql

@@ -48,16 +48,16 @@ select '-- distinct with constants columns';
 select distinct 1 as x, 2 as y from distinct_in_order;
 select distinct 1 as x, 2 as y from distinct_in_order order by x;
 select distinct 1 as x, 2 as y from distinct_in_order order by x, y;
-select distinct a, 1 as x from distinct_in_order order by x;
+select a, x from (select distinct a, 1 as x from distinct_in_order order by x) order by a;
 select distinct a, 1 as x, 2 as y from distinct_in_order order by a;
-select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a;
+select a, b, x, y from(select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a) order by a, b;
 select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by y;
-select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by b;
+select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by a, b;
 -- { echoOff }
 
 drop table if exists distinct_in_order sync;
 
-select '-- check that distinct in order has the same result as ordinary distinct';
+select '-- check that distinct in order returns the same result as ordinary distinct';
 drop table if exists distinct_cardinality_low sync;
 CREATE TABLE distinct_cardinality_low (low UInt64, medium UInt64, high UInt64) ENGINE MergeTree() ORDER BY (low, medium);
 INSERT INTO distinct_cardinality_low SELECT number % 1e1, number % 1e2, number % 1e3 FROM numbers_mt(1e4);

tests/queries/0_stateless/02361_fsync_profile_events.sh

@@ -0,0 +1,73 @@
+#!/usr/bin/env bash
+# Tags: no-s3-storage
+# Tag no-s3-storage: s3 does not have fsync
+
+CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+# shellcheck source=../shell_config.sh
+. "$CUR_DIR"/../shell_config.sh
+
+$CLICKHOUSE_CLIENT -nm -q "
+    drop table if exists data_fsync_pe;
+
+    create table data_fsync_pe (key Int) engine=MergeTree()
+    order by key
+    settings
+        min_rows_for_wide_part=2,
+        fsync_after_insert=1,
+        fsync_part_directory=1;
+"
+
+ret=1
+# Retry in case of fsync/fdatasync was too fast
+# (FileSyncElapsedMicroseconds/DirectorySyncElapsedMicroseconds was 0)
+for i in {1..100}; do
+    query_id="insert-$i-$CLICKHOUSE_DATABASE"
+
+    $CLICKHOUSE_CLIENT --query_id "$query_id" -q "insert into data_fsync_pe values (1)"
+
+    read -r FileSync FileOpen DirectorySync FileSyncElapsedMicroseconds DirectorySyncElapsedMicroseconds <<<"$(
+    $CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q "
+        system flush logs;
+
+        select
+            ProfileEvents['FileSync'],
+            ProfileEvents['FileOpen'],
+            ProfileEvents['DirectorySync'],
+            ProfileEvents['FileSyncElapsedMicroseconds']>0,
+            ProfileEvents['DirectorySyncElapsedMicroseconds']>0
+        from system.query_log
+        where
+            event_date >= yesterday() and
+            current_database = currentDatabase() and
+            query_id = {query_id:String} and
+            type = 'QueryFinish';
+    ")"
+
+    # Non retriable errors
+    if [[ $FileSync -ne 7 ]]; then
+        exit 2
+    fi
+    # Check that all files was synced
+    if [[ $FileSync -ne $FileOpen ]]; then
+        exit 3
+    fi
+    if [[ $DirectorySync -ne 2 ]]; then
+        exit 4
+    fi
+
+    # Retriable errors
+    if [[ $FileSyncElapsedMicroseconds -eq 0 ]]; then
+        continue
+    fi
+    if [[ $DirectorySyncElapsedMicroseconds -eq 0 ]]; then
+        continue
+    fi
+
+    # Everything is OK
+    ret=0
+    break
+done
+
+$CLICKHOUSE_CLIENT -q "drop table data_fsync_pe"
+
+exit $ret