diff --git a/src/Disks/DiskWebServer.h b/src/Disks/DiskWebServer.h index 6341b582174..98f92fe5986 100644 --- a/src/Disks/DiskWebServer.h +++ b/src/Disks/DiskWebServer.h @@ -20,7 +20,7 @@ namespace ErrorCodes * * * web - * https://clickhouse-datasets.s3.yandex.net/disk-with-static-files-tests/test-hits/ + * https://clickhouse-datasets.s3.amazonaws.com/disk-with-static-files-tests/test-hits/ * * * diff --git a/utils/clickhouse-diagnostics/clickhouse-diagnostics b/utils/clickhouse-diagnostics/clickhouse-diagnostics index 2fe67071c3c..cf65e4efbfb 100755 --- a/utils/clickhouse-diagnostics/clickhouse-diagnostics +++ b/utils/clickhouse-diagnostics/clickhouse-diagnostics @@ -665,7 +665,7 @@ class DiagnosticsData: def _dump_wiki(self): """ - Dump diagnostic data in Yandex wiki format. + Dump diagnostic data in Wiki format. """ def _write_title(buffer, value): diff --git a/utils/graphite-rollup/rollup-tag-list.xml b/utils/graphite-rollup/rollup-tag-list.xml index ef28f2089ad..edab2f16436 100644 --- a/utils/graphite-rollup/rollup-tag-list.xml +++ b/utils/graphite-rollup/rollup-tag-list.xml @@ -1,4 +1,4 @@ - + plain @@ -164,4 +164,4 @@ - + diff --git a/utils/graphite-rollup/rollup-typed.xml b/utils/graphite-rollup/rollup-typed.xml index 0b27d43ece9..ace439dba4a 100644 --- a/utils/graphite-rollup/rollup-typed.xml +++ b/utils/graphite-rollup/rollup-typed.xml @@ -1,4 +1,4 @@ - + plain @@ -164,4 +164,4 @@ - + diff --git a/utils/graphite-rollup/rollup.xml b/utils/graphite-rollup/rollup.xml index 641b0130509..2089605c8bf 100644 --- a/utils/graphite-rollup/rollup.xml +++ b/utils/graphite-rollup/rollup.xml @@ -1,4 +1,4 @@ - + \.sum$ @@ -144,4 +144,4 @@ - + diff --git a/utils/release/push_packages b/utils/release/push_packages deleted file mode 100755 index e25cb325c71..00000000000 --- a/utils/release/push_packages +++ /dev/null @@ -1,250 +0,0 @@ -#!/usr/bin/env python3 -# -*- coding: utf-8 -*- -import argparse -import subprocess -import os -import logging -import shutil -import base64 -import pexpect - - -# Do nothing if keys are not provided -class GpgKey(object): - gnupg_dir = os.path.expanduser('~/.gnupg') - TEMPGNUPG_DIR = os.path.expanduser('~/.local/tempgnupg') - - def __init__(self, secret_key_path, public_key_path): - if secret_key_path and public_key_path: - with open(secret_key_path, 'r') as sec, open(public_key_path, 'r') as pub: - self._secret_key = sec.read() - self._public_key = pub.read() - else: - self._secret_key = None - self._public_key = None - - def __enter__(self): - if self._secret_key and self._public_key: - if os.path.exists(self.gnupg_dir): - shutil.move(self.gnupg_dir, self.TEMPGNUPG_DIR) - os.mkdir(self.gnupg_dir) - open(os.path.join(self.gnupg_dir, 'secring.gpg'), 'wb').write(base64.b64decode(self._secret_key)) - open(os.path.join(self.gnupg_dir, 'pubring.gpg'), 'wb').write(base64.b64decode(self._public_key)) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if self._secret_key and self._public_key: - shutil.rmtree(self.gnupg_dir) - if os.path.exists(self.TEMPGNUPG_DIR): - shutil.move(self.TEMPGNUPG_DIR, self.gnupg_dir) - - -class DebRelease(object): - - DUPLOAD_CONF_TEMPLATE = '\n\t'.join(( - "$cfg{{'{title}'}} = {{", - 'fqdn => "{fqdn}",', - 'method => "{method}",', - 'login => "{login}",', - 'incoming => "{incoming}",', - 'options => "{options}",', - 'dinstall_runs => {dinstall_runs},\n}};',)) - DUPLOAD_CONF_PATH = os.path.expanduser('~/.dupload.conf') - DUPLOAD_CONF_TMP_PATH = os.path.expanduser('~/.local/tmp_dupload.cnf') - - def __init__(self, dupload_config, login, ssh_key_path): - self.__config = {} - for repo, conf in dupload_config.items(): - d = { - "fqdn": conf["fqdn"], - "method": "scpb", - "login": login, - "incoming": conf["incoming"], - "dinstall_runs": 0, - "options": "-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ConnectionAttempts=3", - } - d.update(conf) - self.__config[repo] = d - print(self.__config) - self.ssh_key_path = ssh_key_path - - def __enter__(self): - if os.path.exists(self.DUPLOAD_CONF_PATH): - shutil.move(self.DUPLOAD_CONF_PATH, self.DUPLOAD_CONF_TMP_PATH) - self.__dupload_conf = open(self.DUPLOAD_CONF_PATH, 'w') - self.__dupload_conf.write('package config;\n\n$default_host = undef;\n\n' + '\n\n'.join([ - self.DUPLOAD_CONF_TEMPLATE.format(title=title, **values) - for title, values in self.__config.items()])) - self.__dupload_conf.write('\n') - self.__dupload_conf.close() - if self.ssh_key_path: - subprocess.check_call("ssh-add {}".format(self.ssh_key_path), shell=True) - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - if os.path.exists(self.DUPLOAD_CONF_TMP_PATH): - shutil.move(self.DUPLOAD_CONF_TMP_PATH, self.DUPLOAD_CONF_PATH) - else: - os.unlink(self.DUPLOAD_CONF_PATH) - - -class SSHConnection(object): - def __init__(self, user, host, ssh_key=None): - if ssh_key: - key_str = "-i {}".format(ssh_key) - else: - key_str = "" - - self.base_cmd = "ssh {key} {user}@{host}".format( - key=key_str, user=user, host=host) - - def execute(self, cmd): - logging.info("Executing remote cmd %s", cmd) - subprocess.check_call(self.base_cmd + ' "{cmd}"'.format(cmd=cmd), - shell=True) - - -def debsign(path, gpg_passphrase, gpg_sec_key_path, gpg_pub_key_path, gpg_user): - try: - with GpgKey(gpg_sec_key_path, gpg_pub_key_path): - cmd = ('debsign -k \'{key}\' -p"gpg --verbose --no-use-agent --batch ' - '--no-tty --passphrase {passphrase}" {path}/*.changes').format( - key=gpg_user, passphrase=gpg_passphrase, path=path) - logging.info("Build debsign cmd '%s'", cmd) - subprocess.check_call(cmd, shell=True) - logging.info("debsign finished") - except Exception as ex: - logging.error("Cannot debsign packages on path %s, with user key", path) - raise ex - -def rpmsign(path, gpg_passphrase, gpg_sec_key_path, gpg_pub_key_path, gpg_user): - try: - with GpgKey(gpg_sec_key_path, gpg_pub_key_path): - for package in os.listdir(path): - package_path = os.path.join(path, package) - logging.info("Signing %s", package_path) - proc = pexpect.spawn('rpm --resign -D "_signature gpg" -D "_gpg_name {username}" {package}'.format(username=gpg_user, package=package_path)) - proc.expect_exact("Enter pass phrase: ") - proc.sendline(gpg_passphrase) - proc.expect(pexpect.EOF) - logging.info("Signed successfully") - except Exception as ex: - logging.error("Cannot rpmsign packages on path %s, with user key", path) - raise ex - -def transfer_packages_scp(ssh_key, path, repo_user, repo_url, incoming_directory): - logging.info("Transferring packages via scp to %s", repo_url) - if ssh_key: - key_str = "-i {}".format(ssh_key) - else: - key_str = "" - subprocess.check_call('scp {key_str} {path}/* {user}@{repo}:{incoming}'.format( - path=path, user=repo_user, repo=repo_url, key_str=key_str, incoming=incoming_directory), shell=True) - logging.info("Transfer via scp finished") - -def transfer_packages_dupload(ssh_key, path, repo_user, repo_url, incoming_directory): - repo_short_name = repo_url.split('.')[0] - config = { - repo_short_name: { - "fqdn": repo_url, - "incoming": incoming_directory, - } - } - with DebRelease(config, repo_user, ssh_key): - logging.info("Duploading") - subprocess.check_call("dupload -f --nomail --to {repo} {path}".format(repo=repo_short_name, path=path), shell=True) - logging.info("Dupload finished") - - -def clear_old_incoming_packages(ssh_connection, user): - for pkg in ('deb', 'rpm', 'tgz'): - for release_type in ('stable', 'testing', 'prestable', 'lts'): - try: - ssh_connection.execute("rm /home/{user}/incoming/clickhouse/{pkg}/{release_type}/*".format( - user=user, pkg=pkg, release_type=release_type)) - except Exception: - logging.info("rm is not required") - - -def _get_incoming_path(repo_url, user=None, pkg_type=None, release_type=None): - if repo_url == 'repo.mirror.yandex.net': - return "/home/{user}/incoming/clickhouse/{pkg}/{release_type}".format( - user=user, pkg=pkg_type, release_type=release_type) - else: - return "/repo/{0}/mini-dinstall/incoming/".format(repo_url.split('.')[0]) - - -def _fix_args(args): - - if args.gpg_sec_key_path and not os.path.isabs(args.gpg_sec_key_path): - args.gpg_sec_key_path = os.path.join(os.getcwd(), args.gpg_sec_key_path) - - if args.gpg_pub_key_path and not os.path.isabs(args.gpg_pub_key_path): - args.gpg_pub_key_path = os.path.join(os.getcwd(), args.gpg_pub_key_path) - - if args.ssh_key_path and not os.path.isabs(args.ssh_key_path): - args.ssh_key_path = os.path.join(os.getcwd(), args.ssh_key_path) - - -if __name__ == "__main__": - logging.basicConfig(level=logging.INFO, format='%(asctime)s %(message)s') - parser = argparse.ArgumentParser(description="Program to push clickhouse packages to repository") - parser.add_argument('--deb-directory') - parser.add_argument('--rpm-directory') - parser.add_argument('--tgz-directory') - parser.add_argument('--release-type', choices=('testing', 'stable', 'prestable', 'lts'), default='testing') - parser.add_argument('--ssh-key-path') - parser.add_argument('--gpg-passphrase', required=True) - parser.add_argument('--gpg-sec-key-path') - parser.add_argument('--gpg-pub-key-path') - parser.add_argument('--gpg-key-user', default='robot-clickhouse') - parser.add_argument('--repo-url', default='repo.mirror.yandex.net') - parser.add_argument('--repo-user', default='buildfarm') - - args = parser.parse_args() - if args.deb_directory is None and args.rpm_directory is None and args.tgz_directory is None: - parser.error('At least one package directory required') - - _fix_args(args) - - is_open_source = args.repo_url == 'repo.mirror.yandex.net' - ssh_connection = SSHConnection(args.repo_user, args.repo_url, args.ssh_key_path) - - packages = [] - if args.deb_directory: - debsign(args.deb_directory, args.gpg_passphrase, args.gpg_sec_key_path, args.gpg_pub_key_path, args.gpg_key_user) - packages.append((args.deb_directory, 'deb')) - - if args.rpm_directory: - if not is_open_source: - raise Exception("Cannot upload .rpm package to {}".format(args.repo_url)) - rpmsign(args.rpm_directory, args.gpg_passphrase, args.gpg_sec_key_path, args.gpg_pub_key_path, args.gpg_key_user) - packages.append((args.rpm_directory, 'rpm')) - - if args.tgz_directory: - if not is_open_source: - raise Exception("Cannot upload .tgz package to {}".format(args.repo_url)) - packages.append((args.tgz_directory, 'tgz')) - - if is_open_source: - logging.info("Clearing old directory with incoming packages on buildfarm") - clear_old_incoming_packages(ssh_connection, args.repo_user) - logging.info("Incoming directory cleared") - - for package_path, package_type in packages: - logging.info("Processing path '%s' with package type %s", package_path, package_type) - incoming_directory = _get_incoming_path(args.repo_url, args.repo_user, package_type, args.release_type) - if package_type == "deb": - transfer_packages_dupload(args.ssh_key_path, package_path, args.repo_user, args.repo_url, incoming_directory) - else: - transfer_packages_scp(args.ssh_key_path, package_path, args.repo_user, args.repo_url, incoming_directory) - - logging.info("Running clickhouse install (it takes about (20-30 minutes)") - ssh_connection.execute("sudo /usr/sbin/ya-clickhouse-{0}-install".format(package_type)) - logging.info("Clickhouse installed") - logging.info("Pushing clickhouse to repo") - ssh_connection.execute("/usr/sbin/push2publicrepo.sh clickhouse") - logging.info("Push finished") - logging.info("Package '%s' pushed", package_type) - else: - transfer_packages_dupload(args.ssh_key_path, args.deb_directory, args.repo_user, args.repo_url, _get_incoming_path(args.repo_url)) diff --git a/utils/s3tools/s3uploader b/utils/s3tools/s3uploader index 0627b19ea8f..33db76f57f4 100755 --- a/utils/s3tools/s3uploader +++ b/utils/s3tools/s3uploader @@ -121,8 +121,8 @@ if __name__ == "__main__": parser = argparse.ArgumentParser( description="Simple tool for uploading datasets to clickhouse S3", usage='%(prog)s [options] {}'.format(USAGE_EXAMPLES)) - parser.add_argument('--s3-api-url', default='s3.mds.yandex.net') - parser.add_argument('--s3-common-url', default='s3.yandex.net') + parser.add_argument('--s3-api-url', default='s3.amazonaws.com') + parser.add_argument('--s3-common-url', default='s3.amazonaws.com') parser.add_argument('--bucket-name', default='clickhouse-datasets') parser.add_argument('--dataset-name', required=True, help='Name of dataset, will be used in uploaded path') diff --git a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp index 8550675cb9e..5c694ee04ef 100644 --- a/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp +++ b/utils/zookeeper-adjust-block-numbers-to-parts/main.cpp @@ -214,7 +214,7 @@ try po::options_description desc("Allowed options"); desc.add_options() ("help,h", "show help") - ("zookeeper,z", po::value(), "Addresses of ZooKeeper instances, comma-separated. Example: example01e.yandex.ru:2181") + ("zookeeper,z", po::value(), "Addresses of ZooKeeper instances, comma-separated. Example: example01e.clickhouse.com:2181") ("path,p", po::value(), "[optional] Path of replica queue to insert node (without trailing slash). By default it's /clickhouse/tables") ("shard,s", po::value(), "[optional] Shards to process, comma-separated. If not specified then the utility will process all the shards.") ("table,t", po::value(), "[optional] Tables to process, comma-separated. If not specified then the utility will process all the tables.") diff --git a/utils/zookeeper-create-entry-to-download-part/main.cpp b/utils/zookeeper-create-entry-to-download-part/main.cpp index 1e86fe248ab..b92857929b7 100644 --- a/utils/zookeeper-create-entry-to-download-part/main.cpp +++ b/utils/zookeeper-create-entry-to-download-part/main.cpp @@ -11,7 +11,7 @@ try desc.add_options() ("help,h", "produce help message") ("address,a", boost::program_options::value()->required(), - "addresses of ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") + "addresses of ZooKeeper instances, comma separated. Example: example01e.clickhouse.com:2181") ("path,p", boost::program_options::value()->required(), "path of replica queue to insert node (without trailing slash)") ("name,n", boost::program_options::value()->required(), "name of part to download") ; diff --git a/utils/zookeeper-dump-tree/main.cpp b/utils/zookeeper-dump-tree/main.cpp index 893056564bb..d85762df640 100644 --- a/utils/zookeeper-dump-tree/main.cpp +++ b/utils/zookeeper-dump-tree/main.cpp @@ -14,7 +14,7 @@ int main(int argc, char ** argv) desc.add_options() ("help,h", "produce help message") ("address,a", boost::program_options::value()->required(), - "addresses of ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") + "addresses of ZooKeeper instances, comma separated. Example: example01e.clickhouse.com:2181") ("path,p", boost::program_options::value()->default_value("/"), "where to start") ("ctime,c", "print node ctime") diff --git a/utils/zookeeper-remove-by-list/main.cpp b/utils/zookeeper-remove-by-list/main.cpp index 2c97ffb4a70..cf194bd4861 100644 --- a/utils/zookeeper-remove-by-list/main.cpp +++ b/utils/zookeeper-remove-by-list/main.cpp @@ -12,7 +12,7 @@ try desc.add_options() ("help,h", "produce help message") ("address,a", boost::program_options::value()->required(), - "addresses of ZooKeeper instances, comma separated. Example: example01e.yandex.ru:2181") + "addresses of ZooKeeper instances, comma separated. Example: example01e.clickhouse.com:2181") ; boost::program_options::variables_map options;