diff --git a/.github/workflows/master.yml b/.github/workflows/master.yml index fa046ed40c7..2acc1468328 100644 --- a/.github/workflows/master.yml +++ b/.github/workflows/master.yml @@ -151,8 +151,8 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - SplitBuildSmokeTest: - needs: [BuilderDebSplitted] + SharedBuildSmokeTest: + needs: [BuilderDebShared] runs-on: [self-hosted, style-checker] steps: - name: Set envs @@ -171,7 +171,7 @@ jobs: uses: actions/download-artifact@v2 with: path: ${{ env.REPORTS_PATH }} - - name: Split build check + - name: Shared build check run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" @@ -598,7 +598,7 @@ jobs: ########################################################################################## ##################################### SPECIAL BUILDS ##################################### ########################################################################################## - BuilderDebSplitted: + BuilderDebShared: needs: [DockerHubPush] runs-on: [self-hosted, builder] steps: @@ -609,7 +609,7 @@ jobs: IMAGES_PATH=${{runner.temp}}/images_path REPO_COPY=${{runner.temp}}/build_check/ClickHouse CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_splitted + BUILD_NAME=binary_shared EOF - name: Download changed images uses: actions/download-artifact@v2 @@ -1012,7 +1012,7 @@ jobs: # - BuilderBinGCC - BuilderBinPPC64 - BuilderBinClangTidy - - BuilderDebSplitted + - BuilderDebShared runs-on: [self-hosted, style-checker] steps: - name: Set envs @@ -3153,7 +3153,7 @@ jobs: - UnitTestsMsan - UnitTestsUBsan - UnitTestsReleaseClang - - SplitBuildSmokeTest + - SharedBuildSmokeTest runs-on: [self-hosted, style-checker] steps: - name: Clear repository diff --git a/.github/workflows/pull_request.yml b/.github/workflows/pull_request.yml index 26726302beb..3d1ebe1fb8b 100644 --- a/.github/workflows/pull_request.yml +++ b/.github/workflows/pull_request.yml @@ -216,8 +216,8 @@ jobs: # shellcheck disable=SC2046 docker rm -f $(docker ps -a -q) ||: sudo rm -fr "$TEMP_PATH" - SplitBuildSmokeTest: - needs: [BuilderDebSplitted] + SharedBuildSmokeTest: + needs: [BuilderDebShared] runs-on: [self-hosted, style-checker] steps: - name: Set envs @@ -236,7 +236,7 @@ jobs: uses: actions/download-artifact@v2 with: path: ${{ env.REPORTS_PATH }} - - name: Split build check + - name: Shared build check run: | sudo rm -fr "$TEMP_PATH" mkdir -p "$TEMP_PATH" @@ -620,7 +620,7 @@ jobs: ########################################################################################## ##################################### SPECIAL BUILDS ##################################### ########################################################################################## - BuilderDebSplitted: + BuilderDebShared: needs: [DockerHubPush, FastTest, StyleCheck] runs-on: [self-hosted, builder] steps: @@ -631,7 +631,7 @@ jobs: IMAGES_PATH=${{runner.temp}}/images_path REPO_COPY=${{runner.temp}}/build_check/ClickHouse CACHES_PATH=${{runner.temp}}/../ccaches - BUILD_NAME=binary_splitted + BUILD_NAME=binary_shared EOF - name: Download changed images uses: actions/download-artifact@v2 @@ -1024,7 +1024,7 @@ jobs: # - BuilderBinGCC - BuilderBinPPC64 - BuilderBinClangTidy - - BuilderDebSplitted + - BuilderDebShared runs-on: [self-hosted, style-checker] if: ${{ success() || failure() }} steps: @@ -3416,7 +3416,7 @@ jobs: - UnitTestsMsan - UnitTestsUBsan - UnitTestsReleaseClang - - SplitBuildSmokeTest + - SharedBuildSmokeTest - CompatibilityCheck - IntegrationTestsFlakyCheck - Jepsen diff --git a/README.md b/README.md index 93d2da1e39d..20340883853 100644 --- a/README.md +++ b/README.md @@ -15,4 +15,4 @@ ClickHouse® is an open-source column-oriented database management system that a * [Contacts](https://clickhouse.com/company/contact) can help to get your questions answered if there are any. ## Upcoming events -* **v22.8 Release Webinar** Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. +* [**v22.8 Release Webinar**](https://clickhouse.com/company/events/v22-8-release-webinar) Original creator, co-founder, and CTO of ClickHouse Alexey Milovidov will walk us through the highlights of the release, provide live demos, and share vision into what is coming in the roadmap. diff --git a/contrib/arrow b/contrib/arrow index 3e03c6de41a..efdcd015cfd 160000 --- a/contrib/arrow +++ b/contrib/arrow @@ -1 +1 @@ -Subproject commit 3e03c6de41a86df2fc54a61e9be1abaefeff6b0e +Subproject commit efdcd015cfdee1b6aa349c9ca227ca12c3d697f5 diff --git a/docker/packager/packager b/docker/packager/packager index 16a56e9766f..3b9d181d3e0 100755 --- a/docker/packager/packager +++ b/docker/packager/packager @@ -100,12 +100,12 @@ def run_docker_image_with_env( subprocess.check_call(cmd, shell=True) -def is_release_build(build_type, package_type, sanitizer, split_binary): +def is_release_build(build_type, package_type, sanitizer, shared_libraries): return ( build_type == "" and package_type == "deb" and sanitizer == "" - and not split_binary + and not shared_libraries ) @@ -116,7 +116,7 @@ def parse_env_variables( package_type, cache, distcc_hosts, - split_binary, + shared_libraries, clang_tidy, version, author, @@ -202,7 +202,7 @@ def parse_env_variables( cmake_flags.append("-DCMAKE_INSTALL_PREFIX=/usr") cmake_flags.append("-DCMAKE_INSTALL_SYSCONFDIR=/etc") cmake_flags.append("-DCMAKE_INSTALL_LOCALSTATEDIR=/var") - if is_release_build(build_type, package_type, sanitizer, split_binary): + if is_release_build(build_type, package_type, sanitizer, shared_libraries): cmake_flags.append("-DSPLIT_DEBUG_SYMBOLS=ON") result.append("WITH_PERFORMANCE=1") if is_cross_arm: @@ -219,7 +219,7 @@ def parse_env_variables( if package_type == "coverity": result.append("COMBINED_OUTPUT=coverity") result.append('COVERITY_TOKEN="$COVERITY_TOKEN"') - elif split_binary: + elif shared_libraries: result.append("COMBINED_OUTPUT=shared_build") if sanitizer: @@ -264,7 +264,7 @@ def parse_env_variables( result.append("BINARY_OUTPUT=tests") cmake_flags.append("-DENABLE_TESTS=1") - if split_binary: + if shared_libraries: cmake_flags.append( "-DUSE_STATIC_LIBRARIES=0 -DSPLIT_SHARED_LIBRARIES=1" ) @@ -351,7 +351,7 @@ if __name__ == "__main__": default="", ) - parser.add_argument("--split-binary", action="store_true") + parser.add_argument("--shared-libraries", action="store_true") parser.add_argument("--clang-tidy", action="store_true") parser.add_argument("--cache", choices=("ccache", "distcc", ""), default="") parser.add_argument( @@ -404,7 +404,7 @@ if __name__ == "__main__": args.package_type, args.cache, args.distcc_hosts, - args.split_binary, + args.shared_libraries, args.clang_tidy, args.version, args.author, diff --git a/docs/en/operations/server-configuration-parameters/settings.md b/docs/en/operations/server-configuration-parameters/settings.md index d4110714edf..936a20c5e9c 100644 --- a/docs/en/operations/server-configuration-parameters/settings.md +++ b/docs/en/operations/server-configuration-parameters/settings.md @@ -982,7 +982,7 @@ Default value: 2. **Example** ```xml -3 +3 ``` ## background_move_pool_size {#background_move_pool_size} diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 580e5f94952..453ed9ec37c 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -25,6 +25,10 @@ M(WriteBufferFromFileDescriptorWrite, "Number of writes (write/pwrite) to a file descriptor. Does not include sockets.") \ M(WriteBufferFromFileDescriptorWriteFailed, "Number of times the write (write/pwrite) to a file descriptor have failed.") \ M(WriteBufferFromFileDescriptorWriteBytes, "Number of bytes written to file descriptors. If the file is compressed, this will show compressed data size.") \ + M(FileSync, "Number of times the F_FULLFSYNC/fsync/fdatasync function was called for files.") \ + M(DirectorySync, "Number of times the F_FULLFSYNC/fsync/fdatasync function was called for directories.") \ + M(FileSyncElapsedMicroseconds, "Total time spent waiting for F_FULLFSYNC/fsync/fdatasync syscall for files.") \ + M(DirectorySyncElapsedMicroseconds, "Total time spent waiting for F_FULLFSYNC/fsync/fdatasync syscall for directories.") \ M(ReadCompressedBytes, "Number of bytes (the number of bytes before decompression) read from compressed sources (files, network).") \ M(CompressedReadBufferBlocks, "Number of compressed blocks (the blocks of data that are compressed independent of each other) read from compressed sources (files, network).") \ M(CompressedReadBufferBytes, "Number of uncompressed bytes (the number of bytes after decompression) read from compressed sources (files, network).") \ diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index 459c4c44668..5f73352a4c9 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -34,6 +34,8 @@ namespace ProfileEvents extern const Event AIOWriteBytes; extern const Event AIORead; extern const Event AIOReadBytes; + extern const Event FileSync; + extern const Event FileSyncElapsedMicroseconds; } namespace DB @@ -544,6 +546,9 @@ public: file_path, std::to_string(bytes_written)); + ProfileEvents::increment(ProfileEvents::FileSync); + + Stopwatch watch; #if defined(OS_DARWIN) if (::fsync(file.fd) < 0) throwFromErrnoWithPath("Cannot fsync " + file_path, file_path, ErrorCodes::CANNOT_FSYNC); @@ -551,6 +556,7 @@ public: if (::fdatasync(file.fd) < 0) throwFromErrnoWithPath("Cannot fdatasync " + file_path, file_path, ErrorCodes::CANNOT_FSYNC); #endif + ProfileEvents::increment(ProfileEvents::FileSyncElapsedMicroseconds, watch.elapsedMicroseconds()); current_block_index += buffer_size_in_blocks; diff --git a/src/Disks/LocalDirectorySyncGuard.cpp b/src/Disks/LocalDirectorySyncGuard.cpp index 2610cd7c37f..843d1c1ed43 100644 --- a/src/Disks/LocalDirectorySyncGuard.cpp +++ b/src/Disks/LocalDirectorySyncGuard.cpp @@ -1,6 +1,8 @@ #include +#include #include #include +#include #include // O_RDWR /// OSX does not have O_DIRECTORY @@ -8,6 +10,12 @@ #define O_DIRECTORY O_RDWR #endif +namespace ProfileEvents +{ + extern const Event DirectorySync; + extern const Event DirectorySyncElapsedMicroseconds; +} + namespace DB { @@ -29,8 +37,12 @@ LocalDirectorySyncGuard::LocalDirectorySyncGuard(const String & full_path) LocalDirectorySyncGuard::~LocalDirectorySyncGuard() { + ProfileEvents::increment(ProfileEvents::DirectorySync); + try { + Stopwatch watch; + #if defined(OS_DARWIN) if (fcntl(fd, F_FULLFSYNC, 0)) throwFromErrno("Cannot fcntl(F_FULLFSYNC)", ErrorCodes::CANNOT_FSYNC); @@ -40,6 +52,8 @@ LocalDirectorySyncGuard::~LocalDirectorySyncGuard() #endif if (-1 == ::close(fd)) throw Exception("Cannot close file", ErrorCodes::CANNOT_CLOSE_FILE); + + ProfileEvents::increment(ProfileEvents::DirectorySyncElapsedMicroseconds, watch.elapsedMicroseconds()); } catch (...) { diff --git a/src/IO/WriteBufferFromFileDescriptor.cpp b/src/IO/WriteBufferFromFileDescriptor.cpp index 25acb71596a..ba49c16c11f 100644 --- a/src/IO/WriteBufferFromFileDescriptor.cpp +++ b/src/IO/WriteBufferFromFileDescriptor.cpp @@ -18,6 +18,8 @@ namespace ProfileEvents extern const Event WriteBufferFromFileDescriptorWriteFailed; extern const Event WriteBufferFromFileDescriptorWriteBytes; extern const Event DiskWriteElapsedMicroseconds; + extern const Event FileSync; + extern const Event FileSyncElapsedMicroseconds; } namespace CurrentMetrics @@ -113,12 +115,18 @@ void WriteBufferFromFileDescriptor::sync() /// If buffer has pending data - write it. next(); + ProfileEvents::increment(ProfileEvents::FileSync); + + Stopwatch watch; + /// Request OS to sync data with storage medium. #if defined(OS_DARWIN) int res = ::fsync(fd); #else int res = ::fdatasync(fd); #endif + ProfileEvents::increment(ProfileEvents::FileSyncElapsedMicroseconds, watch.elapsedMicroseconds()); + if (-1 == res) throwFromErrnoWithPath("Cannot fsync " + getFileName(), getFileName(), ErrorCodes::CANNOT_FSYNC); } diff --git a/tests/ci/build_check.py b/tests/ci/build_check.py index 3976e2ba916..488fd1bbb34 100644 --- a/tests/ci/build_check.py +++ b/tests/ci/build_check.py @@ -39,7 +39,7 @@ def _can_export_binaries(build_config: BuildConfig) -> bool: return False if build_config["bundled"] != "bundled": return False - if build_config["splitted"] == "splitted": + if build_config["libraries"] == "shared": return False if build_config["sanitizer"] != "": return True @@ -68,8 +68,8 @@ def get_packager_cmd( cmd += f" --build-type={build_config['build_type']}" if build_config["sanitizer"]: cmd += f" --sanitizer={build_config['sanitizer']}" - if build_config["splitted"] == "splitted": - cmd += " --split-binary" + if build_config["libraries"] == "shared": + cmd += " --shared-libraries" if build_config["tidy"] == "enable": cmd += " --clang-tidy" diff --git a/tests/ci/build_report_check.py b/tests/ci/build_report_check.py index 4bb7a619b9f..4246673de3e 100644 --- a/tests/ci/build_report_check.py +++ b/tests/ci/build_report_check.py @@ -37,7 +37,7 @@ class BuildResult: build_type, sanitizer, bundled, - splitted, + libraries, status, elapsed_seconds, with_coverage, @@ -46,7 +46,7 @@ class BuildResult: self.build_type = build_type self.sanitizer = sanitizer self.bundled = bundled - self.splitted = splitted + self.libraries = libraries self.status = status self.elapsed_seconds = elapsed_seconds self.with_coverage = with_coverage @@ -91,7 +91,7 @@ def get_failed_report( build_type="unknown", sanitizer="unknown", bundled="unknown", - splitted="unknown", + libraries="unknown", status=message, elapsed_seconds=0, with_coverage=False, @@ -108,7 +108,7 @@ def process_report( build_type=build_config["build_type"], sanitizer=build_config["sanitizer"], bundled=build_config["bundled"], - splitted=build_config["splitted"], + libraries=build_config["libraries"], status="success" if build_report["status"] else "failure", elapsed_seconds=build_report["elapsed_seconds"], with_coverage=False, diff --git a/tests/ci/ci_config.py b/tests/ci/ci_config.py index 10db5d05ad4..5f2832156cb 100644 --- a/tests/ci/ci_config.py +++ b/tests/ci/ci_config.py @@ -14,7 +14,7 @@ CI_CONFIG = { "package_type": "deb", "static_binary_name": "amd64", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "additional_pkgs": True, "tidy": "disable", "with_coverage": False, @@ -25,7 +25,7 @@ CI_CONFIG = { "sanitizer": "", "package_type": "coverity", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, "official": False, @@ -37,7 +37,7 @@ CI_CONFIG = { "package_type": "deb", "static_binary_name": "aarch64", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "additional_pkgs": True, "tidy": "disable", "with_coverage": False, @@ -48,7 +48,7 @@ CI_CONFIG = { "sanitizer": "address", "package_type": "deb", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -58,7 +58,7 @@ CI_CONFIG = { "sanitizer": "undefined", "package_type": "deb", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -68,7 +68,7 @@ CI_CONFIG = { "sanitizer": "thread", "package_type": "deb", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -78,7 +78,7 @@ CI_CONFIG = { "sanitizer": "memory", "package_type": "deb", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -88,7 +88,7 @@ CI_CONFIG = { "sanitizer": "", "package_type": "deb", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -98,7 +98,7 @@ CI_CONFIG = { "sanitizer": "", "package_type": "binary", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -109,17 +109,17 @@ CI_CONFIG = { "package_type": "binary", "static_binary_name": "debug-amd64", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "enable", "with_coverage": False, }, - "binary_splitted": { + "binary_shared": { "compiler": "clang-14", "build_type": "", "sanitizer": "", "package_type": "binary", "bundled": "bundled", - "splitted": "splitted", + "libraries": "shared", "tidy": "disable", "with_coverage": False, }, @@ -130,7 +130,7 @@ CI_CONFIG = { "package_type": "binary", "static_binary_name": "macos", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -140,7 +140,7 @@ CI_CONFIG = { "sanitizer": "", "package_type": "binary", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -151,7 +151,7 @@ CI_CONFIG = { "package_type": "binary", "static_binary_name": "freebsd", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -162,7 +162,7 @@ CI_CONFIG = { "package_type": "binary", "static_binary_name": "macos-aarch64", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -173,7 +173,7 @@ CI_CONFIG = { "package_type": "binary", "static_binary_name": "powerpc64le", "bundled": "bundled", - "splitted": "unsplitted", + "libraries": "static", "tidy": "disable", "with_coverage": False, }, @@ -192,7 +192,7 @@ CI_CONFIG = { ], "ClickHouse special build check": [ "binary_tidy", - "binary_splitted", + "binary_shared", "binary_darwin", "binary_aarch64", "binary_freebsd", @@ -297,7 +297,7 @@ CI_CONFIG = { "required_build": "package_release", }, "Split build smoke test": { - "required_build": "binary_splitted", + "required_build": "binary_shared", }, "Unit tests (release-clang)": { "required_build": "binary_release", diff --git a/tests/ci/report.py b/tests/ci/report.py index 83d89e628a2..d924be5885b 100644 --- a/tests/ci/report.py +++ b/tests/ci/report.py @@ -290,7 +290,7 @@ tr:hover td {{filter: brightness(95%);}} Build type Sanitizer Bundled -Splitted +Libraries Status Build log Build time @@ -335,7 +335,7 @@ def create_build_html_report( row += "{}".format("none") row += "{}".format(build_result.bundled) - row += "{}".format(build_result.splitted) + row += "{}".format(build_result.libraries) if build_result.status: style = _get_status_style(build_result.status) diff --git a/tests/integration/parallel_skip.json b/tests/integration/parallel_skip.json index 8587e8d4120..b5de1d67cbc 100644 --- a/tests/integration/parallel_skip.json +++ b/tests/integration/parallel_skip.json @@ -46,5 +46,19 @@ "test_storage_s3/test.py::test_url_reconnect_in_the_middle", "test_system_metrics/test.py::test_readonly_metrics", "test_system_replicated_fetches/test.py::test_system_replicated_fetches", - "test_zookeeper_config_load_balancing/test.py::test_round_robin" + "test_zookeeper_config_load_balancing/test.py::test_round_robin", + + "test_tlsv1_3/test.py::test_https", + "test_tlsv1_3/test.py::test_https_wrong_cert", + "test_tlsv1_3/test.py::test_https_non_ssl_auth", + "test_tlsv1_3/test.py::test_create_user", + "test_user_ip_restrictions/test.py::test_ipv4", + "test_user_ip_restrictions/test.py::test_ipv6", + "test_ssl_cert_authentication/test.py::test_https", + "test_ssl_cert_authentication/test.py::test_https_wrong_cert", + "test_ssl_cert_authentication/test.py::test_https_non_ssl_auth", + "test_ssl_cert_authentication/test.py::test_create_user", + "test_grpc_protocol_ssl/test.py::test_secure_channel", + "test_grpc_protocol_ssl/test.py::test_insecure_channel", + "test_grpc_protocol_ssl/test.py::test_wrong_client_certificate" ] diff --git a/tests/integration/test_grpc_protocol_ssl/test.py b/tests/integration/test_grpc_protocol_ssl/test.py index 60c3ccd7a9d..3cd08e3cd6a 100644 --- a/tests/integration/test_grpc_protocol_ssl/test.py +++ b/tests/integration/test_grpc_protocol_ssl/test.py @@ -5,7 +5,8 @@ import grpc from helpers.cluster import ClickHouseCluster, run_and_check GRPC_PORT = 9100 -NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +NODE_IP = "10.5.172.77" # Never copy-paste this line SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) DEFAULT_ENCODING = "utf-8" diff --git a/tests/integration/test_ssl_cert_authentication/test.py b/tests/integration/test_ssl_cert_authentication/test.py index 74bd08e9b35..d210c3e58cc 100644 --- a/tests/integration/test_ssl_cert_authentication/test.py +++ b/tests/integration/test_ssl_cert_authentication/test.py @@ -5,7 +5,8 @@ import ssl import os.path HTTPS_PORT = 8443 -NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +NODE_IP = "10.5.172.77" # Never copy-paste this line NODE_IP_WITH_HTTPS_PORT = NODE_IP + ":" + str(HTTPS_PORT) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_tlsv1_3/test.py b/tests/integration/test_tlsv1_3/test.py index 80c9c68eca7..e93774cf11e 100644 --- a/tests/integration/test_tlsv1_3/test.py +++ b/tests/integration/test_tlsv1_3/test.py @@ -5,7 +5,8 @@ import ssl import os.path HTTPS_PORT = 8443 -NODE_IP = "10.5.172.77" # It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +# It's important for the node to work at this IP because 'server-cert.pem' requires that (see server-ext.cnf). +NODE_IP = "10.5.172.77" # Never copy-paste this line NODE_IP_WITH_HTTPS_PORT = NODE_IP + ":" + str(HTTPS_PORT) SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) diff --git a/tests/integration/test_user_ip_restrictions/test.py b/tests/integration/test_user_ip_restrictions/test.py index e41febfa2f5..f38064e50d0 100644 --- a/tests/integration/test_user_ip_restrictions/test.py +++ b/tests/integration/test_user_ip_restrictions/test.py @@ -8,47 +8,56 @@ node_ipv4 = cluster.add_instance( "node_ipv4", main_configs=[], user_configs=["configs/users_ipv4.xml"], - ipv4_address="10.5.172.77", + ipv4_address="10.5.172.77", # Never copy-paste this line ) client_ipv4_ok = cluster.add_instance( - "client_ipv4_ok", main_configs=[], user_configs=[], ipv4_address="10.5.172.10" + "client_ipv4_ok", + main_configs=[], + user_configs=[], + ipv4_address="10.5.172.10", # Never copy-paste this line ) client_ipv4_ok_direct = cluster.add_instance( - "client_ipv4_ok_direct", main_configs=[], user_configs=[], ipv4_address="10.5.173.1" + "client_ipv4_ok_direct", + main_configs=[], + user_configs=[], + ipv4_address="10.5.173.1", # Never copy-paste this line ) client_ipv4_ok_full_mask = cluster.add_instance( "client_ipv4_ok_full_mask", main_configs=[], user_configs=[], - ipv4_address="10.5.175.77", + ipv4_address="10.5.175.77", # Never copy-paste this line ) client_ipv4_bad = cluster.add_instance( - "client_ipv4_bad", main_configs=[], user_configs=[], ipv4_address="10.5.173.10" + "client_ipv4_bad", + main_configs=[], + user_configs=[], + ipv4_address="10.5.173.10", # Never copy-paste this line ) node_ipv6 = cluster.add_instance( "node_ipv6", main_configs=["configs/config_ipv6.xml"], user_configs=["configs/users_ipv6.xml"], - ipv6_address="2001:3984:3989::1:1000", + ipv6_address="2001:3984:3989::1:1000", # Never copy-paste this line ) client_ipv6_ok = cluster.add_instance( "client_ipv6_ok", main_configs=[], user_configs=[], - ipv6_address="2001:3984:3989::5555", + ipv6_address="2001:3984:3989::5555", # Never copy-paste this line ) client_ipv6_ok_direct = cluster.add_instance( "client_ipv6_ok_direct", main_configs=[], user_configs=[], - ipv6_address="2001:3984:3989::1:1111", + ipv6_address="2001:3984:3989::1:1111", # Never copy-paste this line ) client_ipv6_bad = cluster.add_instance( "client_ipv6_bad", main_configs=[], user_configs=[], - ipv6_address="2001:3984:3989::1:1112", + ipv6_address="2001:3984:3989::1:1112", # Never copy-paste this line ) diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference index efc9e28bcce..d827dc33838 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.reference @@ -85,11 +85,11 @@ select distinct 1 as x, 2 as y from distinct_in_order order by x; 1 2 select distinct 1 as x, 2 as y from distinct_in_order order by x, y; 1 2 -select distinct a, 1 as x from distinct_in_order order by x; +select a, x from (select distinct a, 1 as x from distinct_in_order order by x) order by a; 0 1 select distinct a, 1 as x, 2 as y from distinct_in_order order by a; 0 1 2 -select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a; +select a, b, x, y from(select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a) order by a, b; 0 0 1 2 0 1 1 2 0 2 1 2 @@ -97,10 +97,10 @@ select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a; 0 4 1 2 select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by y; 1 2 -select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by b; +select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by a, b; 0 0 1 2 0 1 1 2 0 2 1 2 0 3 1 2 0 4 1 2 --- check that distinct in order has the same result as ordinary distinct +-- check that distinct in order returns the same result as ordinary distinct diff --git a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql index 7a70e2ef873..0fb3766edb4 100644 --- a/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql +++ b/tests/queries/0_stateless/02317_distinct_in_order_optimization.sql @@ -48,16 +48,16 @@ select '-- distinct with constants columns'; select distinct 1 as x, 2 as y from distinct_in_order; select distinct 1 as x, 2 as y from distinct_in_order order by x; select distinct 1 as x, 2 as y from distinct_in_order order by x, y; -select distinct a, 1 as x from distinct_in_order order by x; +select a, x from (select distinct a, 1 as x from distinct_in_order order by x) order by a; select distinct a, 1 as x, 2 as y from distinct_in_order order by a; -select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a; +select a, b, x, y from(select distinct a, b, 1 as x, 2 as y from distinct_in_order order by a) order by a, b; select distinct x, y from (select 1 as x, 2 as y from distinct_in_order order by x) order by y; -select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by b; +select distinct a, b, x, y from (select a, b, 1 as x, 2 as y from distinct_in_order order by a) order by a, b; -- { echoOff } drop table if exists distinct_in_order sync; -select '-- check that distinct in order has the same result as ordinary distinct'; +select '-- check that distinct in order returns the same result as ordinary distinct'; drop table if exists distinct_cardinality_low sync; CREATE TABLE distinct_cardinality_low (low UInt64, medium UInt64, high UInt64) ENGINE MergeTree() ORDER BY (low, medium); INSERT INTO distinct_cardinality_low SELECT number % 1e1, number % 1e2, number % 1e3 FROM numbers_mt(1e4); diff --git a/tests/queries/0_stateless/02361_fsync_profile_events.reference b/tests/queries/0_stateless/02361_fsync_profile_events.reference new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tests/queries/0_stateless/02361_fsync_profile_events.sh b/tests/queries/0_stateless/02361_fsync_profile_events.sh new file mode 100755 index 00000000000..d54da9a49e5 --- /dev/null +++ b/tests/queries/0_stateless/02361_fsync_profile_events.sh @@ -0,0 +1,73 @@ +#!/usr/bin/env bash +# Tags: no-s3-storage +# Tag no-s3-storage: s3 does not have fsync + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -nm -q " + drop table if exists data_fsync_pe; + + create table data_fsync_pe (key Int) engine=MergeTree() + order by key + settings + min_rows_for_wide_part=2, + fsync_after_insert=1, + fsync_part_directory=1; +" + +ret=1 +# Retry in case of fsync/fdatasync was too fast +# (FileSyncElapsedMicroseconds/DirectorySyncElapsedMicroseconds was 0) +for i in {1..100}; do + query_id="insert-$i-$CLICKHOUSE_DATABASE" + + $CLICKHOUSE_CLIENT --query_id "$query_id" -q "insert into data_fsync_pe values (1)" + + read -r FileSync FileOpen DirectorySync FileSyncElapsedMicroseconds DirectorySyncElapsedMicroseconds <<<"$( + $CLICKHOUSE_CLIENT -nm --param_query_id "$query_id" -q " + system flush logs; + + select + ProfileEvents['FileSync'], + ProfileEvents['FileOpen'], + ProfileEvents['DirectorySync'], + ProfileEvents['FileSyncElapsedMicroseconds']>0, + ProfileEvents['DirectorySyncElapsedMicroseconds']>0 + from system.query_log + where + event_date >= yesterday() and + current_database = currentDatabase() and + query_id = {query_id:String} and + type = 'QueryFinish'; + ")" + + # Non retriable errors + if [[ $FileSync -ne 7 ]]; then + exit 2 + fi + # Check that all files was synced + if [[ $FileSync -ne $FileOpen ]]; then + exit 3 + fi + if [[ $DirectorySync -ne 2 ]]; then + exit 4 + fi + + # Retriable errors + if [[ $FileSyncElapsedMicroseconds -eq 0 ]]; then + continue + fi + if [[ $DirectorySyncElapsedMicroseconds -eq 0 ]]; then + continue + fi + + # Everything is OK + ret=0 + break +done + +$CLICKHOUSE_CLIENT -q "drop table data_fsync_pe" + +exit $ret