thevar1able/ClickHouse
1
0
Fork 0
mirror of https://github.com/ClickHouse/ClickHouse.git synced 2024-11-10 01:25:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' into docs-staging

Browse Source
This commit is contained in:
rfraposa 2022-04-15 07:25:27 -06:00
commit fb08c5b7e1
202 changed files with 4087 additions and 1395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
.github/workflows/backport_branches.yml vendored
View File

@ -155,8 +155,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -196,8 +196,8 @@ jobs:
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -241,8 +241,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -286,8 +286,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -331,8 +331,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |

68
.github/workflows/master.yml vendored
View File

@ -219,8 +219,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -260,8 +260,8 @@ jobs:
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -305,8 +305,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -350,8 +350,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -395,8 +395,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -440,8 +440,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -485,8 +485,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -530,8 +530,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -575,8 +575,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -620,8 +620,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -668,8 +668,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -713,8 +713,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -758,8 +758,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -803,8 +803,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -848,8 +848,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -893,8 +893,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -938,8 +938,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |

2
.github/workflows/nightly.yml vendored
View File

@ -112,7 +112,7 @@ jobs:
run: |
curl --form token="${COVERITY_TOKEN}" \
--form email='security+coverity@clickhouse.com' \
--form file="@$TEMP_PATH/$BUILD_NAME/clickhouse-scan.tgz" \
--form file="@$TEMP_PATH/$BUILD_NAME/coverity-scan.tgz" \
--form version="${GITHUB_REF#refs/heads/}-${GITHUB_SHA::6}" \
--form description="Nighly Scan: $(date +'%Y-%m-%dT%H:%M:%S')" \
https://scan.coverity.com/builds?project=ClickHouse%2FClickHouse

85
.github/workflows/pull_request.yml vendored
View File

@ -153,13 +153,19 @@ jobs:
EOF
- name: Clear repository
run: |
sudo rm -fr "$GITHUB_WORKSPACE" && mkdir "$GITHUB_WORKSPACE"
- name: Check out repository code
uses: actions/checkout@v2
- name: Fast Test
run: |
sudo rm -fr "$GITHUB_WORKSPACE"
mkdir "$GITHUB_WORKSPACE"
sudo rm -fr "$TEMP_PATH"
mkdir -p "$TEMP_PATH"
- name: Check out repository code
uses: actions/checkout@v2
- name: Download changed images
uses: actions/download-artifact@v2
with:
name: changed_images
path: ${{ env.TEMP_PATH }}
- name: Fast Test
run: |
cp -r "$GITHUB_WORKSPACE" "$TEMP_PATH"
cd "$REPO_COPY/tests/ci" && python3 fast_test_check.py
- name: Cleanup
@ -272,8 +278,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -317,8 +323,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -362,8 +368,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -404,8 +410,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -446,8 +452,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -491,8 +497,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -536,8 +542,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -581,8 +587,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -626,8 +632,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -671,8 +677,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -719,8 +725,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -764,8 +770,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -809,8 +815,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -854,8 +860,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -899,8 +905,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -944,8 +950,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -989,8 +995,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -1052,7 +1058,6 @@ jobs:
cat >> "$GITHUB_ENV" << 'EOF'
CHECK_NAME=ClickHouse build check (actions)
REPORTS_PATH=${{runner.temp}}/reports_dir
REPORTS_PATH=${{runner.temp}}/reports_dir
TEMP_PATH=${{runner.temp}}/report_check
EOF
- name: Download json reports

28
.github/workflows/release_branches.yml vendored
View File

@ -146,8 +146,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -187,8 +187,8 @@ jobs:
- name: Upload build URLs to artifacts
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ runner.temp }}/build_check/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -232,8 +232,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -277,8 +277,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -322,8 +322,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -367,8 +367,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |
@ -412,8 +412,8 @@ jobs:
if: ${{ success() || failure() }}
uses: actions/upload-artifact@v2
with:
name: ${{ env.BUILD_NAME }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_NAME }}.json
name: ${{ env.BUILD_URLS }}
path: ${{ env.TEMP_PATH }}/${{ env.BUILD_URLS }}.json
- name: Cleanup
if: always()
run: |

2
cmake/strip_binary.cmake
View File

@ -46,7 +46,7 @@ macro(clickhouse_make_empty_debug_info_for_nfpm)
add_custom_command(TARGET ${EMPTY_DEBUG_TARGET} POST_BUILD
COMMAND mkdir -p "${EMPTY_DEBUG_DESTINATION_DIR}/lib/debug"
COMMAND touch "${EMPTY_DEBUG_DESTINATION_DIR}/lib/debug/${EMPTY_DEBUG_TARGET}.debug"
COMMENT "Addiding empty debug info for NFPM" VERBATIM
COMMENT "Adding empty debug info for NFPM" VERBATIM
)
install(FILES "${EMPTY_DEBUG_DESTINATION_DIR}/lib/debug/${EMPTY_DEBUG_TARGET}.debug" DESTINATION "${CMAKE_INSTALL_LIBDIR}/debug/${CMAKE_INSTALL_FULL_BINDIR}" COMPONENT clickhouse)

1
docker/test/fasttest/run.sh
View File

@ -115,6 +115,7 @@ function start_server
function clone_root
{
git config --global --add safe.directory "$FASTTEST_SOURCE"
git clone --depth 1 https://github.com/ClickHouse/ClickHouse.git -- "$FASTTEST_SOURCE" 2>&1 | ts '%Y-%m-%d %H:%M:%S' | tee "$FASTTEST_OUTPUT/clone_log.txt"
(

8
docker/test/stress/run.sh
View File

@ -329,8 +329,8 @@ then
-e "Code: 1000, e.code() = 111, Connection refused" \
-e "UNFINISHED" \
-e "Renaming unexpected part" \
/var/log/clickhouse-server/clickhouse-server.backward.*.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tOK' >> /test_output/test_results.tsv \
/var/log/clickhouse-server/clickhouse-server.backward.clean.log | zgrep -Fa "<Error>" > /test_output/bc_check_error_messages.txt \
&& echo -e 'Backward compatibility check: Error message in clickhouse-server.log (see bc_check_error_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|| echo -e 'Backward compatibility check: No Error messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
# Remove file bc_check_error_messages.txt if it's empty
@ -346,7 +346,7 @@ then
# OOM
zgrep -Fa " <Fatal> Application: Child process was terminated by signal 9" /var/log/clickhouse-server/clickhouse-server.backward.*.log > /dev/null \
&& echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tOK' >> /test_output/test_results.tsv \
&& echo -e 'Backward compatibility check: OOM killer (or signal 9) in clickhouse-server.log\tFAIL' >> /test_output/test_results.tsv \
|| echo -e 'Backward compatibility check: No OOM messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
# Logical errors
@ -366,7 +366,7 @@ then
# It also checks for crash without stacktrace (printed by watchdog)
echo "Check for Fatal message in server log:"
zgrep -Fa " <Fatal> " /var/log/clickhouse-server/clickhouse-server.backward.*.log > /test_output/bc_check_fatal_messages.txt \
&& echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tOK' >> /test_output/test_results.tsv \
&& echo -e 'Backward compatibility check: Fatal message in clickhouse-server.log (see bc_check_fatal_messages.txt)\tFAIL' >> /test_output/test_results.tsv \
|| echo -e 'Backward compatibility check: No fatal messages in clickhouse-server.log\tOK' >> /test_output/test_results.tsv
# Remove file bc_check_fatal_messages.txt if it's empty

16
docker/test/stress/stress
View File

@ -83,15 +83,15 @@ def make_query_command(query):
def prepare_for_hung_check(drop_databases):
# FIXME this function should not exist, but...
# ThreadFuzzer significantly slows down server and causes false-positive hung check failures
call_with_retry("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'")
# We attach gdb to clickhouse-server before running tests
# to print stacktraces of all crashes even if clickhouse cannot print it for some reason.
# However, it obstruct checking for hung queries.
logging.info("Will terminate gdb (if any)")
call_with_retry("kill -TERM $(pidof gdb)")
# ThreadFuzzer significantly slows down server and causes false-positive hung check failures
call_with_retry("clickhouse client -q 'SYSTEM STOP THREAD FUZZER'")
call_with_retry(make_query_command('SELECT 1 FORMAT Null'))
# Some tests execute SYSTEM STOP MERGES or similar queries.
@ -131,7 +131,7 @@ def prepare_for_hung_check(drop_databases):
Popen(command, shell=True)
break
except Exception as ex:
print("Failed to SHOW or DROP databasese, will retry", ex)
logging.error("Failed to SHOW or DROP databasese, will retry %s", str(ex))
time.sleep(i)
else:
raise Exception("Cannot drop databases after stress tests. Probably server consumed too much memory and cannot execute simple queries")
@ -198,7 +198,11 @@ if __name__ == "__main__":
logging.info("Logs compressed")
if args.hung_check:
have_long_running_queries = prepare_for_hung_check(args.drop_databases)
try:
have_long_running_queries = prepare_for_hung_check(args.drop_databases)
except Exception as ex:
have_long_running_queries = True
logging.error("Failed to prepare for hung check %s", str(ex))
logging.info("Checking if some queries hung")
cmd = ' '.join([args.test_cmd,
# Do not track memory allocations up to 1Gi,
@ -215,6 +219,8 @@ if __name__ == "__main__":
"--client-option", "max_untracked_memory=1Gi",
"--client-option", "max_memory_usage_for_user=0",
"--client-option", "memory_profiler_step=1Gi",
# Use system database to avoid CREATE/DROP DATABASE queries
"--database=system",
"--hung-check",
"00001_select_1"
])

2
docs/en/engines/table-engines/integrations/kafka.md
View File

@ -47,7 +47,7 @@ Optional parameters:
- `kafka_row_delimiter` — Delimiter character, which ends the message.
- `kafka_schema` — Parameter that must be used if the format requires a schema definition. For example, [Cap’n Proto](https://capnproto.org/) requires the path to the schema file and the name of the root `schema.capnp:Message` object.
- `kafka_num_consumers` — The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. The total number of consumers should not exceed the number of partitions in the topic, since only one consumer can be assigned per partition.
- `kafka_num_consumers` — The number of consumers per table. Default: `1`. Specify more consumers if the throughput of one consumer is insufficient. The total number of consumers should not exceed the number of partitions in the topic, since only one consumer can be assigned per partition, and must not be greater than the number of physical cores on the server where ClickHouse is deployed.
- `kafka_max_block_size` — The maximum batch size (in messages) for poll (default: `max_block_size`).
- `kafka_skip_broken_messages` — Kafka message parser tolerance to schema-incompatible messages per block. Default: `0`. If `kafka_skip_broken_messages = N` then the engine skips *N* Kafka messages that cannot be parsed (a message equals a row of data).
- `kafka_commit_every_batch` — Commit every consumed and handled batch instead of a single commit after writing a whole block (default: `0`).

4
docs/en/getting-started/example-datasets/ontime.md
View File

@ -160,6 +160,10 @@ $ clickhouse-client --query "select count(*) from datasets.ontime"
If you will run the queries described below, you have to use the full table name, `datasets.ontime`.
:::
!!! info "Info"
If you are using the prepared partitions or the Online Playground replace any occurrence of `IATA_CODE_Reporting_Airline` or `IATA_CODE_Reporting_Airline AS Carrier` in the following queries with `Carrier` (see `describe ontime`).
## Queries {#queries}
Q0.

9
docs/en/interfaces/cli.md
View File

@ -124,7 +124,7 @@ You can pass parameters to `clickhouse-client` (all parameters have a default va
- `--time, -t` – If specified, print the query execution time to ‘stderr’ in non-interactive mode.
- `--stacktrace` – If specified, also print the stack trace if an exception occurs.
- `--config-file` – The name of the configuration file.
- `--secure` – If specified, will connect to server over secure connection.
- `--secure` – If specified, will connect to server over secure connection (TLS). You might need to configure your CA certificates in the [configuration file](#configuration_files). The available configuration settings are the same as for [server-side TLS configuration](../operations/server-configuration-parameters/settings.md#server_configuration_parameters-openssl).
- `--history_file` — Path to a file containing command history.
- `--param_<name>` — Value for a [query with parameters](#cli-queries-with-parameters).
- `--hardware-utilization` — Print hardware utilization information in progress bar.
@ -148,7 +148,12 @@ Example of a config file:
<config>
<user>username</user>
<password>password</password>
<secure>False</secure>
<secure>true</secure>
<openSSL>
<client>
<caConfig>/etc/ssl/cert.pem</caConfig>
</client>
</openSSL>
</config>
```

2
docs/en/operations/clickhouse-keeper.md
View File

@ -3,7 +3,7 @@ sidebar_position: 66
sidebar_label: ClickHouse Keeper
---
# [pre-production] ClickHouse Keeper {#clickHouse-keeper}
# ClickHouse Keeper {#clickHouse-keeper}
ClickHouse server uses [ZooKeeper](https://zookeeper.apache.org/) coordination system for data [replication](../engines/table-engines/mergetree-family/replication.md) and [distributed DDL](../sql-reference/distributed-ddl.md) queries execution. ClickHouse Keeper is an alternative coordination system compatible with ZooKeeper.

56
docs/en/operations/server-configuration-parameters/settings.md
View File

@ -369,12 +369,12 @@ Opens `https://tabix.io/` when accessing `http://localhost: http_port`.
<http_server_default_response>
<![CDATA[<html ng-app="SMI2"><head><base href="http://ui.tabix.io/"></head><body><div ui-view="" class="content-ui"></div><script src="http://loader.tabix.io/master.js"></script></body></html>]]>
</http_server_default_response>
```
```
## hsts_max_age {#hsts-max-age}
Expired time for HSTS in seconds. The default value is 0 means clickhouse disabled HSTS. If you set a positive number, the HSTS will be enabled and the max-age is the number you set.
**Example**
Expired time for HSTS in seconds. The default value is 0 means clickhouse disabled HSTS. If you set a positive number, the HSTS will be enabled and the max-age is the number you set.
**Example**
```xml
<hsts_max_age>600000</hsts_max_age>
@ -473,7 +473,7 @@ To enable authentication, set `interserver_http_credentials.allow_empty` to `tru
After configuring all replicas set `allow_empty` to `false` or remove this setting. It makes authentication with new credentials mandatory.
To change existing credentials, move the username and the password to `interserver_http_credentials.old` section and update `user` and `password` with new values. At this point the server uses new credentials to connect to other replicas and accepts connections with either new or old credentials.
To change existing credentials, move the username and the password to `interserver_http_credentials.old` section and update `user` and `password` with new values. At this point the server uses new credentials to connect to other replicas and accepts connections with either new or old credentials.
``` xml
<interserver_http_credentials>
@ -842,7 +842,7 @@ The value 0 means that you can delete all tables without any restrictions.
ClickHouse uses threads from the Global Thread pool to process queries. If there is no idle thread to process a query, then a new thread is created in the pool. `max_thread_pool_size` limits the maximum number of threads in the pool.
Possible values:
Possible values:
- Positive integer.
@ -858,7 +858,7 @@ Default value: `10000`.
If the number of **idle** threads in the Global Thread pool is greater than `max_thread_pool_free_size`, then ClickHouse releases resources occupied by some threads and the pool size is decreased. Threads can be created again if necessary.
Possible values:
Possible values:
- Positive integer.
@ -874,7 +874,7 @@ Default value: `1000`.
The maximum number of jobs that can be scheduled on the Global Thread pool. Increasing queue size leads to larger memory usage. It is recommended to keep this value equal to [max_thread_pool_size](#max-thread-pool-size).
Possible values:
Possible values:
- Positive integer.
@ -949,30 +949,30 @@ For more information, see the MergeTreeSettings.h header file.
SSL client/server configuration.
Support for SSL is provided by the `libpoco` library. The interface is described in the file [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h)
Support for SSL is provided by the `libpoco` library. The available configuration options are explained in [SSLManager.h](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/SSLManager.h). Default values can be found in [SSLManager.cpp](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/src/SSLManager.cpp).
Keys for server/client settings:
- privateKeyFile – The path to the file with the secret key of the PEM certificate. The file may contain a key and certificate at the same time.
- certificateFile – The path to the client/server certificate file in PEM format. You can omit it if `privateKeyFile` contains the certificate.
- caConfig – The path to the file or directory that contains trusted root certificates.
- verificationMode – The method for checking the node’s certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: `none`, `relaxed`, `strict`, `once`.
- verificationDepth – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.
- loadDefaultCAFile – Indicates that built-in CA certificates for OpenSSL will be used. Acceptable values: `true`, `false`. \|
- cipherList – Supported OpenSSL encryptions. For example: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`.
- cacheSessions – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Acceptable values: `true`, `false`.
- sessionIdContext – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. This parameter is always recommended since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: `${application.name}`.
- sessionCacheSize – The maximum number of sessions that the server caches. Default value: 1024\*20. 0 – Unlimited sessions.
- sessionTimeout – Time for caching the session on the server.
- extendedVerification – Automatically extended verification of certificates after the session ends. Acceptable values: `true`, `false`.
- requireTLSv1 – Require a TLSv1 connection. Acceptable values: `true`, `false`.
- requireTLSv1_1 – Require a TLSv1.1 connection. Acceptable values: `true`, `false`.
- requireTLSv1_2 – Require a TLSv1.2 connection. Acceptable values: `true`, `false`.
- fips – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS.
- privateKeyPassphraseHandler – Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: `<privateKeyPassphraseHandler>`, `<name>KeyFileHandler</name>`, `<options><password>test</password></options>`, `</privateKeyPassphraseHandler>`.
- invalidCertificateHandler – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: `<invalidCertificateHandler> <name>ConsoleCertificateHandler</name> </invalidCertificateHandler>` .
- disableProtocols – Protocols that are not allowed to use.
- preferServerCiphers – Preferred server ciphers on the client.
- caConfig (default: none) – The path to the file or directory that contains trusted CA certificates. If this points to a file, it must be in PEM format and can contain several CA certificates. If this points to a directory, it must contain one .pem file per CA certificate. The filenames are looked up by the CA subject name hash value. Details can be found in the man page of [SSL_CTX_load_verify_locations](https://www.openssl.org/docs/man3.0/man3/SSL_CTX_load_verify_locations.html).
- verificationMode (default: relaxed) – The method for checking the node’s certificates. Details are in the description of the [Context](https://github.com/ClickHouse-Extras/poco/blob/master/NetSSL_OpenSSL/include/Poco/Net/Context.h) class. Possible values: `none`, `relaxed`, `strict`, `once`.
- verificationDepth (default: 9) – The maximum length of the verification chain. Verification will fail if the certificate chain length exceeds the set value.
- loadDefaultCAFile (default: true) – Wether built-in CA certificates for OpenSSL will be used. ClickHouse assumes that builtin CA certificates are in the file `/etc/ssl/cert.pem` (resp. the directory `/etc/ssl/certs`) or in file (resp. directory) specified by the environment variable `SSL_CERT_FILE` (resp. `SSL_CERT_DIR`).
- cipherList (default: `ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH`) - Supported OpenSSL encryptions.
- cacheSessions (default: false) – Enables or disables caching sessions. Must be used in combination with `sessionIdContext`. Acceptable values: `true`, `false`.
- sessionIdContext (default: `${application.name}`) – A unique set of random characters that the server appends to each generated identifier. The length of the string must not exceed `SSL_MAX_SSL_SESSION_ID_LENGTH`. This parameter is always recommended since it helps avoid problems both if the server caches the session and if the client requested caching. Default value: `${application.name}`.
- sessionCacheSize (default: [1024\*20](https://github.com/ClickHouse/boringssl/blob/master/include/openssl/ssl.h#L1978)) – The maximum number of sessions that the server caches. A value of 0 means unlimited sessions.
- sessionTimeout (default: [2h](https://github.com/ClickHouse/boringssl/blob/master/include/openssl/ssl.h#L1926)) – Time for caching the session on the server.
- extendedVerification (default: false) – If enabled, verify that the certificate CN or SAN matches the peer hostname.
- requireTLSv1 (default: false) – Require a TLSv1 connection. Acceptable values: `true`, `false`.
- requireTLSv1_1 (default: false) – Require a TLSv1.1 connection. Acceptable values: `true`, `false`.
- requireTLSv1_2 (default: false) – Require a TLSv1.2 connection. Acceptable values: `true`, `false`.
- fips (default: false) – Activates OpenSSL FIPS mode. Supported if the library’s OpenSSL version supports FIPS.
- privateKeyPassphraseHandler (default: `KeyConsoleHandler`)– Class (PrivateKeyPassphraseHandler subclass) that requests the passphrase for accessing the private key. For example: `<privateKeyPassphraseHandler>`, `<name>KeyFileHandler</name>`, `<options><password>test</password></options>`, `</privateKeyPassphraseHandler>`.
- invalidCertificateHandler (default: `ConsoleCertificateHandler`) – Class (a subclass of CertificateHandler) for verifying invalid certificates. For example: `<invalidCertificateHandler> <name>ConsoleCertificateHandler</name> </invalidCertificateHandler>` .
- disableProtocols (default: "") – Protocols that are not allowed to use.
- preferServerCiphers (default: false) – Preferred server ciphers on the client.
**Example of settings:**

9
docs/en/operations/storing-data.md
View File

@ -11,10 +11,6 @@ To work with data stored on `Amazon S3` disks use [S3](../engines/table-engines/
To load data from a web server with static files use a disk with type [web](#storing-data-on-webserver).
## Zero-copy Replication {#zero-copy}
ClickHouse supports zero-copy replication for `S3` and `HDFS` disks, which means that if the data is stored remotely on several machines and needs to be synchronized, then only the metadata is replicated (paths to the data parts), but not the data itself.
## Configuring HDFS {#configuring-hdfs}
[MergeTree](../engines/table-engines/mergetree-family/mergetree.md) and [Log](../engines/table-engines/log-family/log.md) family table engines can store data to HDFS using a disk with type `HDFS`.
@ -316,3 +312,8 @@ When loading files by `endpoint`, they must be loaded into `<endpoint>/store/` p
If URL is not reachable on disk load when the server is starting up tables, then all errors are caught. If in this case there were errors, tables can be reloaded (become visible) via `DETACH TABLE table_name` -> `ATTACH TABLE table_name`. If metadata was successfully loaded at server startup, then tables are available straight away.
Use [http_max_single_read_retries](../operations/settings/settings.md#http-max-single-read-retries) setting to limit the maximum number of retries during a single HTTP read.
## Zero-copy Replication (not ready for production) {#zero-copy}
ClickHouse supports zero-copy replication for `S3` and `HDFS` disks, which means that if the data is stored remotely on several machines and needs to be synchronized, then only the metadata is replicated (paths to the data parts), but not the data itself.

2
docs/en/sql-reference/statements/select/index.md
View File

@ -20,7 +20,7 @@ SELECT [DISTINCT [ON (column1, column2, ...)]] expr_list
[WHERE expr]
[GROUP BY expr_list] [WITH ROLLUP|WITH CUBE] [WITH TOTALS]
[HAVING expr]
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr] [INTERPOLATE [(expr_list)]]
[LIMIT [offset_value, ]n BY columns]
[LIMIT [n, ]m] [WITH TIES]
[SETTINGS ...]

60
docs/en/sql-reference/statements/select/order-by.md
View File

@ -280,6 +280,7 @@ To fill multiple columns, add `WITH FILL` modifier with optional parameters afte
``` sql
ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr]
[INTERPOLATE [(col [AS expr], ... colN [AS exprN])]]
```
`WITH FILL` can be applied for fields with Numeric (all kinds of float, decimal, int) or Date/DateTime types. When applied for `String` fields, missed values are filled with empty strings.
@ -287,6 +288,7 @@ When `FROM const_expr` not defined sequence of filling use minimal `expr` field
When `TO const_expr` not defined sequence of filling use maximum `expr` field value from `ORDER BY`.
When `STEP const_numeric_expr` defined then `const_numeric_expr` interprets `as is` for numeric types, as `days` for Date type, as `seconds` for DateTime type. It also supports [INTERVAL](https://clickhouse.com/docs/en/sql-reference/data-types/special-data-types/interval/) data type representing time and date intervals.
When `STEP const_numeric_expr` omitted then sequence of filling use `1.0` for numeric type, `1 day` for Date type and `1 second` for DateTime type.
`INTERPOLATE` can be applied to columns not participating in `ORDER BY WITH FILL`. Such columns are filled based on previous fields values by applying `expr`. If `expr` is not present will repeate previous value. Omitted list will result in including all allowed columns.
Example of a query without `WITH FILL`:
@ -483,4 +485,62 @@ Result:
└────────────┴────────────┴──────────┘
```
Example of a query without `INTERPOLATE`:
``` sql
SELECT n, source, inter FROM (
SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter
FROM numbers(10) WHERE number % 3 = 1
) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5;
```
Result:
``` text
┌───n─┬─source───┬─inter─┐
│ 0 │ │ 0 │
│ 0.5 │ │ 0 │
│ 1 │ original │ 1 │
│ 1.5 │ │ 0 │
│ 2 │ │ 0 │
│ 2.5 │ │ 0 │
│ 3 │ │ 0 │
│ 3.5 │ │ 0 │
│ 4 │ original │ 4 │
│ 4.5 │ │ 0 │
│ 5 │ │ 0 │
│ 5.5 │ │ 0 │
│ 7 │ original │ 7 │
└─────┴──────────┴───────┘
```
Same query after applying `INTERPOLATE`:
``` sql
SELECT n, source, inter FROM (
SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter
FROM numbers(10) WHERE number % 3 = 1
) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5 INTERPOLATE (inter AS inter + 1);
```
Result:
``` text
┌───n─┬─source───┬─inter─┐
│ 0 │ │ 0 │
│ 0.5 │ │ 0 │
│ 1 │ original │ 1 │
│ 1.5 │ │ 2 │
│ 2 │ │ 3 │
│ 2.5 │ │ 4 │
│ 3 │ │ 5 │
│ 3.5 │ │ 6 │
│ 4 │ original │ 4 │
│ 4.5 │ │ 5 │
│ 5 │ │ 6 │
│ 5.5 │ │ 7 │
│ 7 │ original │ 7 │
└─────┴──────────┴───────┘
```
[Original article](https://clickhouse.com/docs/en/sql-reference/statements/select/order-by/) <!--hide-->

2
docs/ru/sql-reference/statements/select/index.md
View File

@ -19,7 +19,7 @@ SELECT [DISTINCT [ON (column1, column2, ...)]] expr_list
[WHERE expr]
[GROUP BY expr_list] [WITH ROLLUP|WITH CUBE] [WITH TOTALS]
[HAVING expr]
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr]
[ORDER BY expr_list] [WITH FILL] [FROM expr] [TO expr] [STEP expr] [INTERPOLATE [(expr_list)]]
[LIMIT [offset_value, ]n BY columns]
[LIMIT [n, ]m] [WITH TIES]
[SETTINGS ...]

58
docs/ru/sql-reference/statements/select/order-by.md
View File

@ -280,6 +280,7 @@ SELECT * FROM collate_test ORDER BY s ASC COLLATE 'en';
```sql
ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_expr], ... exprN [WITH FILL] [FROM expr] [TO expr] [STEP numeric_expr]
[INTERPOLATE [(col [AS expr], ... colN [AS exprN])]]
```
`WITH FILL` может быть применен к полям с числовыми (все разновидности float, int, decimal) или временными (все разновидности Date, DateTime) типами. В случае применения к полям типа `String` недостающие значения заполняются пустой строкой.
@ -289,6 +290,8 @@ ORDER BY expr [WITH FILL] [FROM const_expr] [TO const_expr] [STEP const_numeric_
Когда `STEP const_numeric_expr` не указан, тогда используется `1.0` для числовых типов, `1 день` для типа Date и `1 секунда` для типа DateTime.
`INTERPOLATE` может быть применен к колонкам, не участвующим в `ORDER BY WITH FILL`. Такие колонки заполняются значениями, вычисляемыми применением `expr` к предыдущему значению. Если `expr` опущен, то колонка заполняется предыдущим значением. Если список колонок не указан, то включаются все разрешенные колонки.
Пример запроса без использования `WITH FILL`:
```sql
SELECT n, source FROM (
@ -395,3 +398,58 @@ ORDER BY
│ 1970-03-12 │ 1970-01-08 │ original │
└────────────┴────────────┴──────────┘
```
Пример запроса без `INTERPOLATE`:
``` sql
SELECT n, source, inter FROM (
SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter
FROM numbers(10) WHERE number % 3 = 1
) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5;
```
Результат:
``` text
┌───n─┬─source───┬─inter─┐
│ 0 │ │ 0 │
│ 0.5 │ │ 0 │
│ 1 │ original │ 1 │
│ 1.5 │ │ 0 │
│ 2 │ │ 0 │
│ 2.5 │ │ 0 │
│ 3 │ │ 0 │
│ 3.5 │ │ 0 │
│ 4 │ original │ 4 │
│ 4.5 │ │ 0 │
│ 5 │ │ 0 │
│ 5.5 │ │ 0 │
│ 7 │ original │ 7 │
└─────┴──────────┴───────┘
```
Тот же запрос с `INTERPOLATE`:
``` sql
SELECT n, source, inter FROM (
SELECT toFloat32(number % 10) AS n, 'original' AS source, number as inter
FROM numbers(10) WHERE number % 3 = 1
) ORDER BY n WITH FILL FROM 0 TO 5.51 STEP 0.5 INTERPOLATE (inter AS inter + 1);
```
Результат:
``` text
┌───n─┬─source───┬─inter─┐
│ 0 │ │ 0 │
│ 0.5 │ │ 0 │
│ 1 │ original │ 1 │
│ 1.5 │ │ 2 │
│ 2 │ │ 3 │
│ 2.5 │ │ 4 │
│ 3 │ │ 5 │
│ 3.5 │ │ 6 │
│ 4 │ original │ 4 │
│ 4.5 │ │ 5 │
│ 5 │ │ 6 │
│ 5.5 │ │ 7 │
│ 7 │ original │ 7 │
└─────┴──────────┴───────┘

2
docs/zh/operations/configuration-files.md
View File

@ -3,7 +3,7 @@
ClickHouse支持多配置文件管理。主配置文件是`/etc/clickhouse-server/config.xml`。其余文件须在目录`/etc/clickhouse-server/config.d`。
!!! 注意：
所有配置文件必须是XML格式。此外，配置文件须有相同的跟元素，通常是`<clickhouse>`。
所有配置文件必须是XML格式。此外，配置文件须有相同的根元素，通常是`<clickhouse>`。
主配置文件中的一些配置可以通过`replace`或`remove`属性被配置文件覆盖。

10
src/Backups/ArchiveBackup.cpp
View File

@ -42,6 +42,14 @@ void ArchiveBackup::openImpl(OpenMode open_mode_)
/// mutex is already locked
if (open_mode_ == OpenMode::WRITE)
{
/// Create a directory to contain the archive.
auto dir_path = fs::path(path).parent_path();
if (disk)
disk->createDirectories(dir_path);
else
std::filesystem::create_directories(dir_path);
/// Start writing the archive.
if (disk)
writer = createArchiveWriter(path, disk->writeFile(path));
else
@ -65,7 +73,7 @@ void ArchiveBackup::openImpl(OpenMode open_mode_)
}
}
void ArchiveBackup::closeImpl(bool writing_finalized_)
void ArchiveBackup::closeImpl(const Strings &, bool writing_finalized_)
{
/// mutex is already locked
if (writer && writer->isWritingFile())

2
src/Backups/ArchiveBackup.h
View File

@ -35,7 +35,7 @@ public:
private:
bool backupExists() const override;
void openImpl(OpenMode open_mode_) override;
void closeImpl(bool writing_finalized_) override;
void closeImpl(const Strings & written_files_, bool writing_finalized_) override;
bool supportsWritingInMultipleThreads() const override { return false; }
std::unique_ptr<ReadBuffer> readFileImpl(const String & file_name) const override;
std::unique_ptr<WriteBuffer> writeFileImpl(const String & file_name) override;

26
src/Backups/BackupImpl.cpp
View File

@ -107,6 +107,7 @@ void BackupImpl::open(OpenMode open_mode_)
timestamp = std::time(nullptr);
uuid = UUIDHelpers::generateV4();
writing_finalized = false;
written_files.clear();
}
if (open_mode_ == OpenMode::READ)
@ -145,7 +146,7 @@ void BackupImpl::close()
if (open_mode == OpenMode::NONE)
return;
closeImpl(writing_finalized);
closeImpl(written_files, writing_finalized);
uuid = UUIDHelpers::Nil;
timestamp = 0;
@ -202,9 +203,12 @@ void BackupImpl::writeBackupMetadata()
config->setString(prefix + "checksum", getHexUIntLowercase(info.checksum));
if (info.base_size)
{
config->setUInt(prefix + "base_size", info.base_size);
if (info.base_checksum != info.checksum)
config->setBool(prefix + "use_base", true);
if (info.base_size != info.size)
{
config->setUInt(prefix + "base_size", info.base_size);
config->setString(prefix + "base_checksum", getHexUIntLowercase(info.base_checksum));
}
}
}
++index;
@ -213,6 +217,7 @@ void BackupImpl::writeBackupMetadata()
std::ostringstream stream; // STYLE_CHECK_ALLOW_STD_STRING_STREAM
config->save(stream);
String str = stream.str();
written_files.push_back(".backup");
auto out = writeFileImpl(".backup");
out->write(str.data(), str.size());
}
@ -253,13 +258,14 @@ void BackupImpl::readBackupMetadata()
if (info.size)
{
info.checksum = unhexChecksum(config->getString(prefix + "checksum"));
info.base_size = config->getUInt(prefix + "base_size", 0);
bool use_base = config->getBool(prefix + "use_base", false);
info.base_size = config->getUInt(prefix + "base_size", use_base ? info.size : 0);
if (info.base_size)
{
if (config->has(prefix + "base_checksum"))
info.base_checksum = unhexChecksum(config->getString(prefix + "base_checksum"));
else
if (info.base_size == info.size)
info.base_checksum = info.checksum;
else
info.base_checksum = unhexChecksum(config->getString(prefix + "base_checksum"));
}
}
file_infos.emplace(name, info);
@ -345,11 +351,6 @@ BackupEntryPtr BackupImpl::readFile(const String & file_name) const
return std::make_unique<BackupEntryFromMemory>(nullptr, 0, UInt128{0, 0});
}
auto read_callback = [backup = std::static_pointer_cast<const BackupImpl>(shared_from_this()), file_name]()
{
return backup->readFileImpl(file_name);
};
if (!info.base_size)
{
/// Data goes completely from this backup, the base backup isn't used.
@ -526,6 +527,7 @@ void BackupImpl::writeFile(const String & file_name, BackupEntryPtr entry)
}
/// Copy the entry's data after `copy_pos`.
written_files.push_back(file_name);
auto out = writeFileImpl(file_name);
copyData(*read_buffer, *out);

3
src/Backups/BackupImpl.h
View File

@ -47,7 +47,7 @@ protected:
virtual void openImpl(OpenMode open_mode_) = 0;
OpenMode getOpenModeNoLock() const { return open_mode; }
virtual void closeImpl(bool writing_finalized_) = 0;
virtual void closeImpl(const Strings & written_files_, bool writing_finalized_) = 0;
/// Read a file from the backup.
/// Low level: the function doesn't check base backup or checksums.
@ -86,6 +86,7 @@ private:
std::optional<UUID> base_backup_uuid;
std::map<String, FileInfo> file_infos; /// Should be ordered alphabetically, see listFiles().
std::unordered_map<UInt128, String> file_checksums;
Strings written_files;
bool writing_finalized = false;
};

24
src/Backups/BackupInfo.cpp
View File

@ -1,6 +1,7 @@
#include <Backups/BackupInfo.h>
#include <Parsers/ASTExpressionList.h>
#include <Parsers/ASTFunction.h>
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ExpressionElementParsers.h>
#include <Parsers/formatAST.h>
@ -23,7 +24,11 @@ String BackupInfo::toString() const
auto list = std::make_shared<ASTExpressionList>();
func->arguments = list;
func->children.push_back(list);
list->children.reserve(args.size());
list->children.reserve(args.size() + !id_arg.empty());
if (!id_arg.empty())
list->children.push_back(std::make_shared<ASTIdentifier>(id_arg));
for (const auto & arg : args)
list->children.push_back(std::make_shared<ASTLiteral>(arg));
@ -53,9 +58,22 @@ BackupInfo BackupInfo::fromAST(const IAST & ast)
const auto * list = func->arguments->as<const ASTExpressionList>();
if (!list)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected list, got {}", serializeAST(*func->arguments));
res.args.reserve(list->children.size());
for (const auto & elem : list->children)
size_t index = 0;
if (!list->children.empty())
{
const auto * id = list->children[0]->as<const ASTIdentifier>();
if (id)
{
res.id_arg = id->name();
++index;
}
}
res.args.reserve(list->children.size() - index);
for (; index < list->children.size(); ++index)
{
const auto & elem = list->children[index];
const auto * lit = elem->as<const ASTLiteral>();
if (!lit)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected literal, got {}", serializeAST(*elem));

1
src/Backups/BackupInfo.h
View File

@ -11,6 +11,7 @@ class IAST;
struct BackupInfo
{
String backup_engine_name;
String id_arg;
std::vector<Field> args;
String toString() const;

42
src/Backups/DirectoryBackup.cpp
View File

@ -1,16 +1,9 @@
#include <Backups/DirectoryBackup.h>
#include <Common/quoteString.h>
#include <Disks/IDisk.h>
#include <Disks/DiskLocal.h>
namespace DB
{
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
}
DirectoryBackup::DirectoryBackup(
const String & backup_name_,
@ -19,23 +12,16 @@ DirectoryBackup::DirectoryBackup(
const ContextPtr & context_,
const std::optional<BackupInfo> & base_backup_info_)
: BackupImpl(backup_name_, context_, base_backup_info_)
, disk(disk_), path(path_)
, disk(disk_)
{
/// Path to backup must end with '/'
if (!path.ends_with("/"))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Backup {}: Path to backup must end with '/', but {} doesn't.", getName(), quoteString(path));
dir_path = fs::path(path).parent_path(); /// get path without terminating slash
/// Remove terminating slash.
path = (std::filesystem::path(path_) / "").parent_path();
/// If `disk` is not specified, we create an internal instance of `DiskLocal` here.
if (!disk)
{
auto fspath = fs::path{dir_path};
if (!fspath.has_filename())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Backup {}: Path to a backup must be a directory path.", getName(), quoteString(path));
path = fspath.filename() / "";
dir_path = fs::path(path).parent_path(); /// get path without terminating slash
String disk_path = fspath.remove_filename();
disk = std::make_shared<DiskLocal>(disk_path, disk_path, 0);
disk = std::make_shared<DiskLocal>(path, path, 0);
path = ".";
}
}
@ -47,34 +33,38 @@ DirectoryBackup::~DirectoryBackup()
bool DirectoryBackup::backupExists() const
{
return disk->isDirectory(dir_path);
return disk->isDirectory(path);
}
void DirectoryBackup::openImpl(OpenMode open_mode_)
{
if (open_mode_ == OpenMode::WRITE)
disk->createDirectories(dir_path);
disk->createDirectories(path);
}
void DirectoryBackup::closeImpl(bool writing_finalized_)
void DirectoryBackup::closeImpl(const Strings & written_files_, bool writing_finalized_)
{
if ((getOpenModeNoLock() == OpenMode::WRITE) && !writing_finalized_ && disk->isDirectory(dir_path))
if ((getOpenModeNoLock() == OpenMode::WRITE) && !writing_finalized_ && !written_files_.empty())
{
/// Creating of the backup wasn't finished correctly,
/// so the backup cannot be used and it's better to remove its files.
disk->removeRecursive(dir_path);
const auto & files_to_delete = written_files_;
for (const String & file_name : files_to_delete)
disk->removeFileIfExists(path / file_name);
if (disk->isDirectory(path) && disk->isDirectoryEmpty(path))
disk->removeDirectory(path);
}
}
std::unique_ptr<ReadBuffer> DirectoryBackup::readFileImpl(const String & file_name) const
{
String file_path = path + file_name;
auto file_path = path / file_name;
return disk->readFile(file_path);
}
std::unique_ptr<WriteBuffer> DirectoryBackup::writeFileImpl(const String & file_name)
{
String file_path = path + file_name;
auto file_path = path / file_name;
disk->createDirectories(fs::path(file_path).parent_path());
return disk->writeFile(file_path);
}

6
src/Backups/DirectoryBackup.h
View File

@ -1,6 +1,7 @@
#pragma once
#include <Backups/BackupImpl.h>
#include <filesystem>
namespace DB
@ -25,13 +26,12 @@ public:
private:
bool backupExists() const override;
void openImpl(OpenMode open_mode_) override;
void closeImpl(bool writing_finalized_) override;
void closeImpl(const Strings & written_files_, bool writing_finalized_) override;
std::unique_ptr<ReadBuffer> readFileImpl(const String & file_name) const override;
std::unique_ptr<WriteBuffer> writeFileImpl(const String & file_name) override;
DiskPtr disk;
String path;
String dir_path; /// `path` without terminating slash
std::filesystem::path path;
};
}

139
src/Backups/registerBackupEnginesFileAndDisk.cpp
View File

@ -2,6 +2,7 @@
#include <Backups/DirectoryBackup.h>
#include <Backups/ArchiveBackup.h>
#include <Common/quoteString.h>
#include <Disks/IDisk.h>
#include <IO/Archives/hasRegisteredArchiveFileExtension.h>
#include <Interpreters/Context.h>
#include <Poco/Util/AbstractConfiguration.h>
@ -13,8 +14,9 @@ namespace DB
namespace ErrorCodes
{
extern const int BAD_ARGUMENTS;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int INVALID_CONFIG_PARAMETER;
extern const int LOGICAL_ERROR;
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
}
@ -22,83 +24,71 @@ namespace
{
namespace fs = std::filesystem;
[[noreturn]] void throwDiskIsAllowed(const String & disk_name)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk {} is not allowed for backups", disk_name);
}
[[noreturn]] void throwPathNotAllowed(const fs::path & path)
{
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} is not allowed for backups", quoteString(String{path}));
}
void checkAllowedPathInConfigIsValid(const String & key, const fs::path & value)
{
if (value.empty() || value.is_relative())
throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Configuration parameter {} has a wrong value {}", key, String{value});
}
/// Checks that a disk name and a path specified as parameters of Disk() are valid.
void checkDiskNameAndPath(const String & disk_name, fs::path & path, const Poco::Util::AbstractConfiguration & config)
/// Checks that a disk name specified as parameters of Disk() is valid.
void checkDiskName(const String & disk_name, const Poco::Util::AbstractConfiguration & config)
{
String key = "backups.allowed_disk";
bool disk_name_found = false;
if (!config.has(key))
throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "The \"backups.allowed_disk\" configuration parameter is not set, cannot use Disk() backup engine");
size_t counter = 0;
while (config.has(key))
while (config.getString(key) != disk_name)
{
if (config.getString(key) == disk_name)
{
disk_name_found = true;
break;
}
key = "backups.allowed_disk[" + std::to_string(++counter) + "]";
if (!config.has(key))
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Disk {} is not allowed for backups, see the \"backups.allowed_disk\" configuration parameter", disk_name);
}
if (!disk_name_found)
throwDiskIsAllowed(disk_name);
path = path.lexically_normal();
if (!path.is_relative() || path.empty() || (*path.begin() == ".."))
throwPathNotAllowed(path);
}
/// Checks that a path specified as a parameter of File() is valid.
void checkPath(fs::path & path, const Poco::Util::AbstractConfiguration & config)
/// Checks that a path specified as parameters of Disk() is valid.
void checkPath(const String & disk_name, const DiskPtr & disk, fs::path & path)
{
String key = "backups.allowed_path";
path = path.lexically_normal();
if (!path.is_relative() && (disk->getType() == DiskType::Local))
path = path.lexically_proximate(disk->getPath());
bool path_ok = path.empty() || (path.is_relative() && (*path.begin() != ".."));
if (!path_ok)
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path {} to backup must be inside the specified disk {}", quoteString(path.c_str()), disk_name);
}
/// Checks that a path specified as parameters of File() is valid.
void checkPath(fs::path & path, const Poco::Util::AbstractConfiguration & config, const fs::path & data_dir)
{
path = path.lexically_normal();
if (path.empty())
throwPathNotAllowed(path);
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to backup must not be empty");
String key = "backups.allowed_path";
if (!config.has(key))
throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER,
"The \"backups.allowed_path\" configuration parameter is not set, cannot use File() backup engine");
if (path.is_relative())
{
if (*path.begin() == "..")
throwPathNotAllowed(path);
auto first_allowed_path = fs::path(config.getString(key));
if (first_allowed_path.is_relative())
first_allowed_path = data_dir / first_allowed_path;
auto base = fs::path(config.getString(key, ""));
checkAllowedPathInConfigIsValid(key, base);
path = base / path;
return;
path = first_allowed_path / path;
}
bool path_found_in_config = false;
size_t counter = 0;
while (config.has(key))
while (true)
{
auto base = fs::path(config.getString(key));
checkAllowedPathInConfigIsValid(key, base);
auto rel = path.lexically_relative(base);
if (!rel.empty() && (*rel.begin() != ".."))
{
path_found_in_config = true;
auto allowed_path = fs::path(config.getString(key));
if (allowed_path.is_relative())
allowed_path = data_dir / allowed_path;
auto rel = path.lexically_proximate(allowed_path);
bool path_ok = rel.empty() || (rel.is_relative() && (*rel.begin() != ".."));
if (path_ok)
break;
}
key = "backups.allowed_path[" + std::to_string(++counter) + "]";
if (!config.has(key))
throw Exception(ErrorCodes::BAD_ARGUMENTS,
"Path {} is not allowed for backups, see the \"backups.allowed_path\" configuration parameter",
quoteString(path.c_str()));
}
if (!path_found_in_config)
throwPathNotAllowed(path);
}
}
@ -109,6 +99,15 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
{
String backup_name = params.backup_info.toString();
const String & engine_name = params.backup_info.backup_engine_name;
if (!params.backup_info.id_arg.empty())
{
throw Exception(
ErrorCodes::BAD_ARGUMENTS,
"Backup engine '{}' requires first argument to be a string",
engine_name);
}
const auto & args = params.backup_info.args;
DiskPtr disk;
@ -123,7 +122,9 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
}
path = args[0].safeGet<String>();
checkPath(path, params.context->getConfigRef());
const auto & config = params.context->getConfigRef();
const auto & data_dir = params.context->getPath();
checkPath(path, config, data_dir);
}
else if (engine_name == "Disk")
{
@ -135,30 +136,28 @@ void registerBackupEnginesFileAndDisk(BackupFactory & factory)
}
String disk_name = args[0].safeGet<String>();
const auto & config = params.context->getConfigRef();
checkDiskName(disk_name, config);
path = args[1].safeGet<String>();
checkDiskNameAndPath(disk_name, path, params.context->getConfigRef());
disk = params.context->getDisk(disk_name);
checkPath(disk_name, disk, path);
}
else
throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected backup engine '{}'", engine_name);
std::unique_ptr<IBackup> backup;
if (!path.has_filename() && !path.empty())
{
if (!params.password.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Password is not applicable, backup cannot be encrypted");
backup = std::make_unique<DirectoryBackup>(backup_name, disk, path, params.context, params.base_backup_info);
}
else if (hasRegisteredArchiveFileExtension(path))
if (hasRegisteredArchiveFileExtension(path))
{
auto archive_backup = std::make_unique<ArchiveBackup>(backup_name, disk, path, params.context, params.base_backup_info);
archive_backup->setCompression(params.compression_method, params.compression_level);
archive_backup->setPassword(params.password);
backup = std::move(archive_backup);
return archive_backup;
}
else
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Path to backup must be either a directory or a path to an archive");
return backup;
{
if (!params.password.empty())
throw Exception(ErrorCodes::BAD_ARGUMENTS, "Password is not applicable, backup cannot be encrypted");
return std::make_unique<DirectoryBackup>(backup_name, disk, path, params.context, params.base_backup_info);
}
};
factory.registerBackupEngine("File", creator_fn);

80
src/Client/ClientBase.cpp
View File

@ -137,14 +137,14 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src)
auto & dst_column_host_name = typeid_cast<ColumnString &>(*mutable_columns[name_pos["host_name"]]);
auto & dst_array_current_time = typeid_cast<ColumnUInt32 &>(*mutable_columns[name_pos["current_time"]]).getData();
auto & dst_array_thread_id = typeid_cast<ColumnUInt64 &>(*mutable_columns[name_pos["thread_id"]]).getData();
// auto & dst_array_thread_id = typeid_cast<ColumnUInt64 &>(*mutable_columns[name_pos["thread_id"]]).getData();
auto & dst_array_type = typeid_cast<ColumnInt8 &>(*mutable_columns[name_pos["type"]]).getData();
auto & dst_column_name = typeid_cast<ColumnString &>(*mutable_columns[name_pos["name"]]);
auto & dst_array_value = typeid_cast<ColumnInt64 &>(*mutable_columns[name_pos["value"]]).getData();
const auto & src_column_host_name = typeid_cast<const ColumnString &>(*src.getByName("host_name").column);
const auto & src_array_current_time = typeid_cast<const ColumnUInt32 &>(*src.getByName("current_time").column).getData();
// const auto & src_array_thread_id = typeid_cast<const ColumnUInt64 &>(*src.getByName("thread_id").column).getData();
const auto & src_array_thread_id = typeid_cast<const ColumnUInt64 &>(*src.getByName("thread_id").column).getData();
const auto & src_column_name = typeid_cast<const ColumnString &>(*src.getByName("name").column);
const auto & src_array_value = typeid_cast<const ColumnInt64 &>(*src.getByName("value").column).getData();
@ -169,6 +169,16 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src)
rows_by_name[id] = src_row;
}
/// Filter out snapshots
std::set<size_t> thread_id_filter_mask;
for (size_t i = 0; i < src_array_thread_id.size(); ++i)
{
if (src_array_thread_id[i] != 0)
{
thread_id_filter_mask.emplace(i);
}
}
/// Merge src into dst.
for (size_t dst_row = 0; dst_row < dst_rows; ++dst_row)
{
@ -180,6 +190,11 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src)
if (auto it = rows_by_name.find(id); it != rows_by_name.end())
{
size_t src_row = it->second;
if (thread_id_filter_mask.contains(src_row))
{
continue;
}
dst_array_current_time[dst_row] = src_array_current_time[src_row];
switch (dst_array_type[dst_row])
@ -199,24 +214,18 @@ static void incrementProfileEventsBlock(Block & dst, const Block & src)
/// Copy rows from src that dst does not contains.
for (const auto & [id, pos] : rows_by_name)
{
if (thread_id_filter_mask.contains(pos))
{
continue;
}
for (size_t col = 0; col < src.columns(); ++col)
{
mutable_columns[col]->insert((*src.getByPosition(col).column)[pos]);
}
}
/// Filter out snapshots
std::set<size_t> thread_id_filter_mask;
for (size_t i = 0; i < dst_array_thread_id.size(); ++i)
{
if (dst_array_thread_id[i] != 0)
{
thread_id_filter_mask.emplace(i);
}
}
dst.setColumns(std::move(mutable_columns));
dst.erase(thread_id_filter_mask);
}
@ -225,17 +234,16 @@ std::atomic_flag exit_on_signal;
class QueryInterruptHandler : private boost::noncopyable
{
public:
QueryInterruptHandler() { exit_on_signal.clear(); }
~QueryInterruptHandler() { exit_on_signal.test_and_set(); }
static void start() { exit_on_signal.clear(); }
/// Return true if the query was stopped.
static bool stop() { return exit_on_signal.test_and_set(); }
static bool cancelled() { return exit_on_signal.test(); }
};
/// This signal handler is set only for SIGINT.
void interruptSignalHandler(int signum)
{
if (exit_on_signal.test_and_set())
if (QueryInterruptHandler::stop())
safeExit(128 + signum);
}
@ -254,7 +262,7 @@ ClientBase::ClientBase() = default;
void ClientBase::setupSignalHandler()
{
exit_on_signal.test_and_set();
QueryInterruptHandler::stop();
struct sigaction new_act;
memset(&new_act, 0, sizeof(new_act));
@ -395,8 +403,16 @@ void ClientBase::onData(Block & block, ASTPtr parsed_query)
if (need_render_progress && (stdout_is_a_tty || is_interactive) && !select_into_file)
progress_indication.clearProgressOutput();
output_format->write(materializeBlock(block));
written_first_block = true;
try
{
output_format->write(materializeBlock(block));
written_first_block = true;
}
catch (const Exception &)
{
/// Catch client errors like NO_ROW_DELIMITER
throw LocalFormatError(getCurrentExceptionMessage(print_stack_trace), getCurrentExceptionCode());
}
/// Received data block is immediately displayed to the user.
output_format->flush();
@ -685,6 +701,9 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
{
try
{
QueryInterruptHandler::start();
SCOPE_EXIT({ QueryInterruptHandler::stop(); });
connection->sendQuery(
connection_parameters.timeouts,
query,
@ -724,8 +743,6 @@ void ClientBase::processOrdinaryQuery(const String & query_to_execute, ASTPtr pa
/// Also checks if query execution should be cancelled.
void ClientBase::receiveResult(ASTPtr parsed_query)
{
QueryInterruptHandler query_interrupt_handler;
// TODO: get the poll_interval from commandline.
const auto receive_timeout = connection_parameters.timeouts.receive_timeout;
constexpr size_t default_poll_interval = 1000000; /// in microseconds
@ -760,7 +777,7 @@ void ClientBase::receiveResult(ASTPtr parsed_query)
};
/// handler received sigint
if (query_interrupt_handler.cancelled())
if (QueryInterruptHandler::cancelled())
{
cancel_query();
}
@ -937,6 +954,9 @@ void ClientBase::onProfileEvents(Block & block)
auto elapsed_time = profile_events.watch.elapsedMicroseconds();
progress_indication.updateThreadEventData(thread_times, elapsed_time);
if (need_render_progress)
progress_indication.writeProgress();
if (profile_events.print)
{
if (profile_events.watch.elapsedMilliseconds() >= profile_events.delay_ms)
@ -1413,6 +1433,8 @@ void ClientBase::processParsedSingleQuery(const String & full_query, const Strin
progress_indication.clearProgressOutput();
logs_out_stream->writeProfileEvents(profile_events.last_block);
logs_out_stream->flush();
profile_events.last_block = {};
}
if (is_interactive)
@ -1838,7 +1860,7 @@ void ClientBase::runInteractive()
}
LineReader::Patterns query_extenders = {"\\"};
LineReader::Patterns query_delimiters = {";", "\\G"};
LineReader::Patterns query_delimiters = {";", "\\G", "\\G;"};
#if USE_REPLXX
replxx::Replxx::highlighter_callback_t highlight_callback{};
@ -1860,9 +1882,13 @@ void ClientBase::runInteractive()
break;
has_vertical_output_suffix = false;
if (input.ends_with("\\G"))
if (input.ends_with("\\G") || input.ends_with("\\G;"))
{
input.resize(input.size() - 2);
if (input.ends_with("\\G"))
input.resize(input.size() - 2);
else if (input.ends_with("\\G;"))
input.resize(input.size() - 3);
has_vertical_output_suffix = true;
}

19
src/Client/LocalConnection.cpp
View File

@ -201,9 +201,6 @@ void LocalConnection::finishQuery()
{
next_packet_type = Protocol::Server::EndOfStream;
if (!state)
return;
if (state->executor)
{
state->executor.reset();
@ -219,6 +216,7 @@ void LocalConnection::finishQuery()
state->io.onFinish();
state.reset();
last_sent_snapshots.clear();
}
bool LocalConnection::poll(size_t)
@ -326,6 +324,21 @@ bool LocalConnection::poll(size_t)
}
}
if (state->is_finished && !state->sent_profile_events)
{
state->sent_profile_events = true;
if (send_profile_events && state->executor)
{
Block block;
state->after_send_profile_events.restart();
next_packet_type = Protocol::Server::ProfileEvents;
getProfileEvents(block);
state->block.emplace(std::move(block));
return true;
}
}
if (state->is_finished)
{
finishQuery();

1
src/Client/LocalConnection.h
View File

@ -47,6 +47,7 @@ struct LocalQueryState
bool sent_extremes = false;
bool sent_progress = false;
bool sent_profile_info = false;
bool sent_profile_events = false;
/// To output progress, the difference after the previous sending of progress.
Progress progress;

14
src/Common/Concepts.h Normal file
View File

@ -0,0 +1,14 @@
#pragma once
#include <concepts>
namespace DB
{
template <typename... T>
concept OptionalArgument = requires(T &&...)
{
requires(sizeof...(T) == 0 || sizeof...(T) == 1);
};
}

169
src/Common/FileCache.cpp
View File

@ -233,6 +233,88 @@ FileSegments LRUFileCache::splitRangeIntoCells(
return file_segments;
}
void LRUFileCache::fillHolesWithEmptyFileSegments(
FileSegments & file_segments, const Key & key, const FileSegment::Range & range, bool fill_with_detached_file_segments, std::lock_guard<std::mutex> & cache_lock)
{
/// There are segments [segment1, ..., segmentN]
/// (non-overlapping, non-empty, ascending-ordered) which (maybe partially)
/// intersect with given range.
/// It can have holes:
/// [____________________] -- requested range
/// [____] [_] [_________] -- intersecting cache [segment1, ..., segmentN]
///
/// For each such hole create a cell with file segment state EMPTY.
auto it = file_segments.begin();
auto segment_range = (*it)->range();
size_t current_pos;
if (segment_range.left < range.left)
{
/// [_______ -- requested range
/// [_______
/// ^
/// segment1
current_pos = segment_range.right + 1;
++it;
}
else
current_pos = range.left;
while (current_pos <= range.right && it != file_segments.end())
{
segment_range = (*it)->range();
if (current_pos == segment_range.left)
{
current_pos = segment_range.right + 1;
++it;
continue;
}
assert(current_pos < segment_range.left);
auto hole_size = segment_range.left - current_pos;
if (fill_with_detached_file_segments)
{
auto file_segment = std::make_shared<FileSegment>(current_pos, hole_size, key, this, FileSegment::State::EMPTY);
file_segment->detached = true;
file_segments.insert(it, file_segment);
}
else
{
file_segments.splice(it, splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, cache_lock));
}
current_pos = segment_range.right + 1;
++it;
}
if (current_pos <= range.right)
{
/// ________] -- requested range
/// _____]
/// ^
/// segmentN
auto hole_size = range.right - current_pos + 1;
if (fill_with_detached_file_segments)
{
auto file_segment = std::make_shared<FileSegment>(current_pos, hole_size, key, this, FileSegment::State::EMPTY);
file_segment->detached = true;
file_segments.insert(file_segments.end(), file_segment);
}
else
{
file_segments.splice(file_segments.end(), splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, cache_lock));
}
}
}
FileSegmentsHolder LRUFileCache::getOrSet(const Key & key, size_t offset, size_t size)
{
assertInitialized();
@ -254,69 +336,42 @@ FileSegmentsHolder LRUFileCache::getOrSet(const Key & key, size_t offset, size_t
}
else
{
/// There are segments [segment1, ..., segmentN]
/// (non-overlapping, non-empty, ascending-ordered) which (maybe partially)
/// intersect with given range.
/// It can have holes:
/// [____________________] -- requested range
/// [____] [_] [_________] -- intersecting cache [segment1, ..., segmentN]
///
/// For each such hole create a cell with file segment state EMPTY.
auto it = file_segments.begin();
auto segment_range = (*it)->range();
size_t current_pos;
if (segment_range.left < range.left)
{
/// [_______ -- requested range
/// [_______
/// ^
/// segment1
current_pos = segment_range.right + 1;
++it;
}
else
current_pos = range.left;
while (current_pos <= range.right && it != file_segments.end())
{
segment_range = (*it)->range();
if (current_pos == segment_range.left)
{
current_pos = segment_range.right + 1;
++it;
continue;
}
assert(current_pos < segment_range.left);
auto hole_size = segment_range.left - current_pos;
file_segments.splice(it, splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, cache_lock));
current_pos = segment_range.right + 1;
++it;
}
if (current_pos <= range.right)
{
/// ________] -- requested range
/// _____]
/// ^
/// segmentN
auto hole_size = range.right - current_pos + 1;
file_segments.splice(file_segments.end(), splitRangeIntoCells(key, current_pos, hole_size, FileSegment::State::EMPTY, cache_lock));
}
fillHolesWithEmptyFileSegments(file_segments, key, range, false, cache_lock);
}
assert(!file_segments.empty());
return FileSegmentsHolder(std::move(file_segments));
}
FileSegmentsHolder LRUFileCache::get(const Key & key, size_t offset, size_t size)
{
assertInitialized();
FileSegment::Range range(offset, offset + size - 1);
std::lock_guard cache_lock(mutex);
#ifndef NDEBUG
assertCacheCorrectness(key, cache_lock);
#endif
/// Get all segments which intersect with the given range.
auto file_segments = getImpl(key, range, cache_lock);
if (file_segments.empty())
{
auto file_segment = std::make_shared<FileSegment>(offset, size, key, this, FileSegment::State::EMPTY);
file_segment->detached = true;
file_segments = { file_segment };
}
else
{
fillHolesWithEmptyFileSegments(file_segments, key, range, true, cache_lock);
}
return FileSegmentsHolder(std::move(file_segments));
}
LRUFileCache::FileSegmentCell * LRUFileCache::addCell(
const Key & key, size_t offset, size_t size, FileSegment::State state,
std::lock_guard<std::mutex> & cache_lock)

16
src/Common/FileCache.h
View File

@ -72,6 +72,17 @@ public:
*/
virtual FileSegmentsHolder getOrSet(const Key & key, size_t offset, size_t size) = 0;
/**
* Segments in returned list are ordered in ascending order and represent a full contiguous
* interval (no holes). Each segment in returned list has state: DOWNLOADED, DOWNLOADING or EMPTY.
*
* If file segment has state EMPTY, then it is also marked as "detached". E.g. it is "detached"
* from cache (not owned by cache), and as a result will never change it's state and will be destructed
* with the destruction of the holder, while in getOrSet() EMPTY file segments can eventually change
* it's state (and become DOWNLOADED).
*/
virtual FileSegmentsHolder get(const Key & key, size_t offset, size_t size) = 0;
virtual FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) = 0;
virtual FileSegments getSnapshot() const = 0;
@ -124,6 +135,8 @@ public:
FileSegmentsHolder getOrSet(const Key & key, size_t offset, size_t size) override;
FileSegmentsHolder get(const Key & key, size_t offset, size_t size) override;
FileSegments getSnapshot() const override;
FileSegmentsHolder setDownloading(const Key & key, size_t offset, size_t size) override;
@ -213,6 +226,9 @@ private:
String dumpStructureImpl(const Key & key_, std::lock_guard<std::mutex> & cache_lock);
void fillHolesWithEmptyFileSegments(
FileSegments & file_segments, const Key & key, const FileSegment::Range & range, bool fill_with_detached_file_segments, std::lock_guard<std::mutex> & cache_lock);
public:
struct Stat
{

37
src/Common/FileSegment.cpp
View File

@ -107,6 +107,9 @@ String FileSegment::getOrSetDownloader()
{
std::lock_guard segment_lock(mutex);
if (detached)
throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Cannot set downloader for a detached file segment");
if (downloader_id.empty())
{
assert(download_state != State::DOWNLOADING);
@ -216,6 +219,8 @@ void FileSegment::write(const char * from, size_t size, size_t offset_)
"Attempt to write {} bytes to offset: {}, but current download offset is {}",
size, offset_, download_offset);
assertNotDetached();
if (!cache_writer)
{
if (downloaded_size > 0)
@ -263,6 +268,8 @@ void FileSegment::writeInMemory(const char * from, size_t size)
ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
"Not enough space is reserved. Available: {}, expected: {}", availableSize(), size);
assertNotDetached();
std::lock_guard segment_lock(mutex);
if (cache_writer)
@ -297,7 +304,9 @@ size_t FileSegment::finalizeWrite()
size_t size = cache_writer->offset();
if (size == 0)
throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Writing size is not allowed");
throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Writing zero size is not allowed");
assertNotDetached();
try
{
@ -352,6 +361,8 @@ bool FileSegment::reserve(size_t size)
if (!size)
throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "Zero space reservation is not allowed");
assertNotDetached();
{
std::lock_guard segment_lock(mutex);
@ -419,7 +430,10 @@ void FileSegment::completeBatchAndResetDownloader()
if (!is_downloader)
{
cv.notify_all();
throw Exception(ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR, "File segment can be completed only by downloader");
throw Exception(
ErrorCodes::REMOTE_FS_OBJECT_CACHE_ERROR,
"File segment can be completed only by downloader ({} != {})",
downloader_id, getCallerId());
}
resetDownloaderImpl(segment_lock);
@ -453,6 +467,8 @@ void FileSegment::complete(State state)
download_state = state;
assertNotDetached();
try
{
completeImpl(cache_lock, segment_lock);
@ -479,6 +495,8 @@ void FileSegment::complete(std::lock_guard<std::mutex> & cache_lock)
if (download_state != State::DOWNLOADED && getDownloadedSize(segment_lock) == range().size())
setDownloaded(segment_lock);
assertNotDetached();
if (download_state == State::DOWNLOADING || download_state == State::EMPTY)
{
/// Segment state can be changed from DOWNLOADING or EMPTY only if the caller is the
@ -608,6 +626,12 @@ void FileSegment::assertCorrectnessImpl(std::lock_guard<std::mutex> & /* segment
assert(download_state != FileSegment::State::DOWNLOADED || std::filesystem::file_size(cache->getPathInLocalCache(key(), offset())) > 0);
}
void FileSegment::assertNotDetached() const
{
if (detached)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Operation not allowed, file segment is detached");
}
FileSegmentPtr FileSegment::getSnapshot(const FileSegmentPtr & file_segment, std::lock_guard<std::mutex> & /* cache_lock */)
{
auto snapshot = std::make_shared<FileSegment>(
@ -641,6 +665,15 @@ FileSegmentsHolder::~FileSegmentsHolder()
if (!cache)
cache = file_segment->cache;
if (file_segment->detached)
{
/// This file segment is not owned by cache, so it will be destructed
/// at this point, therefore no completion required.
assert(file_segment->state() == FileSegment::State::EMPTY);
file_segment_it = file_segments.erase(current_file_segment_it);
continue;
}
try
{
/// File segment pointer must be reset right after calling complete() and

3
src/Common/FileSegment.h
View File

@ -150,6 +150,7 @@ private:
size_t getDownloadedSize(std::lock_guard<std::mutex> & segment_lock) const;
String getInfoForLogImpl(std::lock_guard<std::mutex> & segment_lock) const;
void assertCorrectnessImpl(std::lock_guard<std::mutex> & segment_lock) const;
void assertNotDetached() const;
void setDownloaded(std::lock_guard<std::mutex> & segment_lock);
void setDownloadFailed(std::lock_guard<std::mutex> & segment_lock);
@ -199,6 +200,8 @@ private:
Poco::Logger * log;
/// "detached" file segment means that it is not owned by cache ("detached" from cache).
/// In general case, all file segments are owned by cache.
bool detached = false;
std::atomic<bool> is_downloaded{false};

4
src/Common/ThreadPool.cpp
View File

@ -1,4 +1,5 @@
#include <Common/ThreadPool.h>
#include <Common/setThreadName.h>
#include <Common/Exception.h>
#include <Common/getNumberOfPhysicalCPUCores.h>
@ -243,6 +244,9 @@ void ThreadPoolImpl<Thread>::worker(typename std::list<Thread>::iterator thread_
while (true)
{
/// This is inside the loop to also reset previous thread names set inside the jobs.
setThreadName("ThreadPool");
Job job;
bool need_shutdown = false;

2
src/Common/ZooKeeper/ZooKeeperCommon.h
View File

@ -57,6 +57,8 @@ struct ZooKeeperRequest : virtual Request
bool restored_from_zookeeper_log = false;
UInt64 request_created_time_ns = 0;
UInt64 thread_id = 0;
String query_id;
ZooKeeperRequest() = default;
ZooKeeperRequest(const ZooKeeperRequest &) = default;

11
src/Common/ZooKeeper/ZooKeeperImpl.cpp
View File

@ -8,6 +8,7 @@
#include <IO/Operators.h>
#include <IO/WriteBufferFromString.h>
#include <base/logger_useful.h>
#include <base/getThreadId.h>
#include <Common/config.h>
@ -1016,6 +1017,11 @@ void ZooKeeper::pushRequest(RequestInfo && info)
try
{
info.time = clock::now();
if (zk_log)
{
info.request->thread_id = getThreadId();
info.request->query_id = String(CurrentThread::getQueryId());
}
if (!info.request->xid)
{
@ -1269,6 +1275,11 @@ void ZooKeeper::logOperationIfNeeded(const ZooKeeperRequestPtr & request, const
elem.event_time = event_time;
elem.address = socket_address;
elem.session_id = session_id;
if (request)
{
elem.thread_id = request->thread_id;
elem.query_id = request->query_id;
}
maybe_zk_log->add(elem);
}
}

72
src/Common/renameat2.cpp → src/Common/atomicRename.cpp
View File

@ -1,4 +1,4 @@
#include <Common/renameat2.h>
#include <Common/atomicRename.h>
#include <Common/Exception.h>
#include <Common/VersionNumber.h>
#include <Poco/Environment.h>
@ -55,7 +55,7 @@ namespace ErrorCodes
namespace DB
{
static bool supportsRenameat2Impl()
static bool supportsAtomicRenameImpl()
{
VersionNumber renameat2_minimal_version(3, 15, 0);
VersionNumber linux_version(Poco::Environment::osVersion());
@ -64,7 +64,7 @@ static bool supportsRenameat2Impl()
static bool renameat2(const std::string & old_path, const std::string & new_path, int flags)
{
if (!supportsRenameat2())
if (!supportsAtomicRename())
return false;
if (old_path.empty() || new_path.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot rename {} to {}: path is empty", old_path, new_path);
@ -93,9 +93,69 @@ static bool renameat2(const std::string & old_path, const std::string & new_path
throwFromErrnoWithPath(fmt::format("Cannot rename {} to {}", old_path, new_path), new_path, ErrorCodes::SYSTEM_ERROR);
}
bool supportsRenameat2()
bool supportsAtomicRename()
{
static bool supports = supportsRenameat2Impl();
static bool supports = supportsAtomicRenameImpl();
return supports;
}
}
#elif defined(__APPLE__)
// Includes
#include <dlfcn.h> // For dlsym
#include <stdio.h> // For renamex_np
#include <string.h> // For stderror
#ifndef RENAME_SWAP
#define RENAME_SWAP 0x00000002
#endif
#ifndef RENAME_EXCL
#define RENAME_EXCL 0x00000004
#endif
#define RENAME_NOREPLACE RENAME_EXCL
#define RENAME_EXCHANGE RENAME_SWAP
namespace DB
{
static bool renameat2(const std::string & old_path, const std::string & new_path, int flags)
{
using function_type = int (*)(const char * from, const char * to, unsigned int flags);
static function_type fun = reinterpret_cast<function_type>(dlsym(RTLD_DEFAULT, "renamex_np"));
if (fun == nullptr)
return false;
if (old_path.empty() || new_path.empty())
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot rename {} to {}: path is empty", old_path, new_path);
if (0 == (*fun)(old_path.c_str(), new_path.c_str(), flags))
return true;
int errnum = errno;
if (errnum == ENOTSUP || errnum == EINVAL)
return false;
if (errnum == EEXIST)
throwFromErrno(fmt::format("Cannot rename {} to {} because the second path already exists", old_path, new_path), ErrorCodes::ATOMIC_RENAME_FAIL);
if (errnum == ENOENT)
throwFromErrno(fmt::format("Paths cannot be exchanged because {} or {} does not exist", old_path, new_path), ErrorCodes::ATOMIC_RENAME_FAIL);
throwFromErrnoWithPath(
fmt::format("Cannot rename {} to {}: {}", old_path, new_path, strerror(errnum)), new_path, ErrorCodes::SYSTEM_ERROR);
}
static bool supportsAtomicRenameImpl()
{
auto fun = dlsym(RTLD_DEFAULT, "renamex_np");
return fun != nullptr;
}
bool supportsAtomicRename()
{
static bool supports = supportsAtomicRenameImpl();
return supports;
}
@ -114,7 +174,7 @@ static bool renameat2(const std::string &, const std::string &, int)
return false;
}
bool supportsRenameat2()
bool supportsAtomicRename()
{
return false;
}

2
src/Common/renameat2.h → src/Common/atomicRename.h
View File

@ -6,7 +6,7 @@ namespace DB
{
/// Returns true, if the following functions supported by the system
bool supportsRenameat2();
bool supportsAtomicRename();
/// Atomically rename old_path to new_path. If new_path exists, do not overwrite it and throw exception
void renameNoReplace(const std::string & old_path, const std::string & new_path);

13
src/Common/filesystemHelpers.cpp
View File

@ -234,6 +234,11 @@ bool createFile(const std::string & path)
DB::throwFromErrnoWithPath("Cannot create file: " + path, path, DB::ErrorCodes::CANNOT_CREATE_FILE);
}
bool exists(const std::string & path)
{
return faccessat(AT_FDCWD, path.c_str(), F_OK, AT_EACCESS) == 0;
}
bool canRead(const std::string & path)
{
struct stat st;
@ -249,7 +254,6 @@ bool canRead(const std::string & path)
DB::throwFromErrnoWithPath("Cannot check read access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
}
bool canWrite(const std::string & path)
{
struct stat st;
@ -265,6 +269,13 @@ bool canWrite(const std::string & path)
DB::throwFromErrnoWithPath("Cannot check write access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
}
bool canExecute(const std::string & path)
{
if (exists(path))
return faccessat(AT_FDCWD, path.c_str(), X_OK, AT_EACCESS) == 0;
DB::throwFromErrnoWithPath("Cannot check execute access to file: " + path, path, DB::ErrorCodes::PATH_ACCESS_DENIED);
}
time_t getModificationTime(const std::string & path)
{
struct stat st;

2
src/Common/filesystemHelpers.h
View File

@ -70,8 +70,10 @@ namespace FS
{
bool createFile(const std::string & path);
bool exists(const std::string & path);
bool canRead(const std::string & path);
bool canWrite(const std::string & path);
bool canExecute(const std::string & path);
time_t getModificationTime(const std::string & path);
Poco::Timestamp getModificationTimestamp(const std::string & path);

77
src/Common/memory.h
View File

@ -1,8 +1,11 @@
#pragma once
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wold-style-cast"
#include <new>
#include <base/defines.h>
#include <Common/Concepts.h>
#include <Common/CurrentMemoryTracker.h>
#include <Common/config.h>
@ -14,13 +17,24 @@
# include <cstdlib>
#endif
namespace Memory
{
inline ALWAYS_INLINE void * newImpl(std::size_t size)
inline ALWAYS_INLINE size_t alignToSizeT(std::align_val_t align) noexcept
{
auto * ptr = malloc(size);
return static_cast<size_t>(align);
}
template <std::same_as<std::align_val_t>... TAlign>
requires DB::OptionalArgument<TAlign...>
inline ALWAYS_INLINE void * newImpl(std::size_t size, TAlign... align)
{
void * ptr = nullptr;
if constexpr (sizeof...(TAlign) == 1)
ptr = aligned_alloc(alignToSizeT(align...), size);
else
ptr = malloc(size);
if (likely(ptr != nullptr))
return ptr;
@ -33,6 +47,11 @@ inline ALWAYS_INLINE void * newNoExept(std::size_t size) noexcept
return malloc(size);
}
inline ALWAYS_INLINE void * newNoExept(std::size_t size, std::align_val_t align) noexcept
{
return aligned_alloc(static_cast<size_t>(align), size);
}
inline ALWAYS_INLINE void deleteImpl(void * ptr) noexcept
{
free(ptr);
@ -40,17 +59,24 @@ inline ALWAYS_INLINE void deleteImpl(void * ptr) noexcept
#if USE_JEMALLOC
inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size) noexcept
template <std::same_as<std::align_val_t>... TAlign>
requires DB::OptionalArgument<TAlign...>
inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size, TAlign... align) noexcept
{
if (unlikely(ptr == nullptr))
return;
sdallocx(ptr, size, 0);
if constexpr (sizeof...(TAlign) == 1)
sdallocx(ptr, size, MALLOCX_ALIGN(alignToSizeT(align...)));
else
sdallocx(ptr, size, 0);
}
#else
inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]]) noexcept
template <std::same_as<std::align_val_t>... TAlign>
requires DB::OptionalArgument<TAlign...>
inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unused]], TAlign... /* align */) noexcept
{
free(ptr);
}
@ -58,13 +84,14 @@ inline ALWAYS_INLINE void deleteSized(void * ptr, std::size_t size [[maybe_unuse
#endif
#if defined(OS_LINUX)
# include <malloc.h>
# include <malloc.h>
#elif defined(OS_DARWIN)
# include <malloc/malloc.h>
# include <malloc/malloc.h>
#endif
inline ALWAYS_INLINE size_t getActualAllocationSize(size_t size)
template <std::same_as<std::align_val_t>... TAlign>
requires DB::OptionalArgument<TAlign...>
inline ALWAYS_INLINE size_t getActualAllocationSize(size_t size, TAlign... align [[maybe_unused]])
{
size_t actual_size = size;
@ -72,26 +99,41 @@ inline ALWAYS_INLINE size_t getActualAllocationSize(size_t size)
/// The nallocx() function allocates no memory, but it performs the same size computation as the mallocx() function
/// @note je_mallocx() != je_malloc(). It's expected they don't differ much in allocation logic.
if (likely(size != 0))
actual_size = nallocx(size, 0);
{
if constexpr (sizeof...(TAlign) == 1)
actual_size = nallocx(size, MALLOCX_ALIGN(alignToSizeT(align...)));
else
actual_size = nallocx(size, 0);
}
#endif
return actual_size;
}
inline ALWAYS_INLINE void trackMemory(std::size_t size)
template <std::same_as<std::align_val_t>... TAlign>
requires DB::OptionalArgument<TAlign...>
inline ALWAYS_INLINE void trackMemory(std::size_t size, TAlign... align)
{
std::size_t actual_size = getActualAllocationSize(size);
std::size_t actual_size = getActualAllocationSize(size, align...);
CurrentMemoryTracker::allocNoThrow(actual_size);
}
inline ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t size [[maybe_unused]] = 0) noexcept
template <std::same_as<std::align_val_t>... TAlign>
requires DB::OptionalArgument<TAlign...>
inline ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t size [[maybe_unused]] = 0, TAlign... align [[maybe_unused]]) noexcept
{
try
{
#if USE_JEMALLOC
/// @note It's also possible to use je_malloc_usable_size() here.
if (likely(ptr != nullptr))
CurrentMemoryTracker::free(sallocx(ptr, 0));
{
if constexpr (sizeof...(TAlign) == 1)
CurrentMemoryTracker::free(sallocx(ptr, MALLOCX_ALIGN(alignToSizeT(align...))));
else
CurrentMemoryTracker::free(sallocx(ptr, 0));
}
#else
if (size)
CurrentMemoryTracker::free(size);
@ -103,7 +145,10 @@ inline ALWAYS_INLINE void untrackMemory(void * ptr [[maybe_unused]], std::size_t
#endif
}
catch (...)
{}
{
}
}
}
#pragma GCC diagnostic pop

54
src/Common/new_delete.cpp
View File

@ -1,6 +1,7 @@
#include <Common/memory.h>
#include <Common/config.h>
#include <cassert>
#include <new>
#include <Common/config.h>
#include <Common/memory.h>
#if defined(OS_DARWIN) && (USE_JEMALLOC)
/// In case of OSX jemalloc register itself as a default zone allocator.
@ -53,12 +54,24 @@ void * operator new(std::size_t size)
return Memory::newImpl(size);
}
void * operator new(std::size_t size, std::align_val_t align)
{
Memory::trackMemory(size, align);
return Memory::newImpl(size, align);
}
void * operator new[](std::size_t size)
{
Memory::trackMemory(size);
return Memory::newImpl(size);
}
void * operator new[](std::size_t size, std::align_val_t align)
{
Memory::trackMemory(size, align);
return Memory::newImpl(size, align);
}
void * operator new(std::size_t size, const std::nothrow_t &) noexcept
{
Memory::trackMemory(size);
@ -71,6 +84,18 @@ void * operator new[](std::size_t size, const std::nothrow_t &) noexcept
return Memory::newNoExept(size);
}
void * operator new(std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept
{
Memory::trackMemory(size, align);
return Memory::newNoExept(size, align);
}
void * operator new[](std::size_t size, std::align_val_t align, const std::nothrow_t &) noexcept
{
Memory::trackMemory(size, align);
return Memory::newNoExept(size, align);
}
/// delete
/// C++17 std 21.6.2.1 (11)
@ -81,26 +106,51 @@ void * operator new[](std::size_t size, const std::nothrow_t &) noexcept
/// It's unspecified whether size-aware or size-unaware version is called when deleting objects of
/// incomplete type and arrays of non-class and trivially-destructible class types.
void operator delete(void * ptr) noexcept
{
Memory::untrackMemory(ptr);
Memory::deleteImpl(ptr);
}
void operator delete(void * ptr, std::align_val_t align) noexcept
{
Memory::untrackMemory(ptr, 0, align);
Memory::deleteImpl(ptr);
}
void operator delete[](void * ptr) noexcept
{
Memory::untrackMemory(ptr);
Memory::deleteImpl(ptr);
}
void operator delete[](void * ptr, std::align_val_t align) noexcept
{
Memory::untrackMemory(ptr, 0, align);
Memory::deleteImpl(ptr);
}
void operator delete(void * ptr, std::size_t size) noexcept
{
Memory::untrackMemory(ptr, size);
Memory::deleteSized(ptr, size);
}
void operator delete(void * ptr, std::size_t size, std::align_val_t align) noexcept
{
Memory::untrackMemory(ptr, size, align);
Memory::deleteSized(ptr, size, align);
}
void operator delete[](void * ptr, std::size_t size) noexcept
{
Memory::untrackMemory(ptr, size);
Memory::deleteSized(ptr, size);
}
void operator delete[](void * ptr, std::size_t size, std::align_val_t align) noexcept
{
Memory::untrackMemory(ptr, size, align);
Memory::deleteSized(ptr, size, align);
}

2
src/Compression/CompressedReadBufferFromFile.cpp
View File

@ -62,7 +62,7 @@ void CompressedReadBufferFromFile::seek(size_t offset_in_compressed_file, size_t
{
/// Nothing to do if we already at required position
if (!size_compressed && static_cast<size_t>(file_in.getPosition()) == offset_in_compressed_file && /// correct position in compressed file
(offset() == offset_in_decompressed_block /// correct position in buffer or
((!buffer().empty() && offset() == offset_in_decompressed_block) /// correct position in buffer or
|| nextimpl_working_buffer_offset == offset_in_decompressed_block)) /// we will move our position to correct one
return;

6
src/Coordination/tests/gtest_coordination.cpp
View File

@ -407,7 +407,7 @@ TEST_P(CoordinationTest, ChangelogTestCompaction)
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
changelog.compact(6);
std::this_thread::sleep_for(std::chrono::microseconds(200));
std::this_thread::sleep_for(std::chrono::microseconds(1000));
EXPECT_FALSE(fs::exists("./logs/changelog_1_5.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_6_10.bin" + params.extension));
@ -1469,7 +1469,7 @@ TEST_P(CoordinationTest, TestRotateIntervalChanges)
}
changelog_2.compact(105);
std::this_thread::sleep_for(std::chrono::microseconds(200));
std::this_thread::sleep_for(std::chrono::microseconds(1000));
EXPECT_FALSE(fs::exists("./logs/changelog_1_100.bin" + params.extension));
EXPECT_TRUE(fs::exists("./logs/changelog_101_110.bin" + params.extension));
@ -1489,7 +1489,7 @@ TEST_P(CoordinationTest, TestRotateIntervalChanges)
}
changelog_3.compact(125);
std::this_thread::sleep_for(std::chrono::microseconds(200));
std::this_thread::sleep_for(std::chrono::microseconds(1000));
EXPECT_FALSE(fs::exists("./logs/changelog_101_110.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_111_117.bin" + params.extension));
EXPECT_FALSE(fs::exists("./logs/changelog_118_124.bin" + params.extension));

32
src/Core/InterpolateDescription.cpp Normal file
View File

@ -0,0 +1,32 @@
#include <Core/Block.h>
#include <IO/Operators.h>
#include <Common/JSONBuilder.h>
#include <Core/InterpolateDescription.h>
#include <Interpreters/convertFieldToType.h>
namespace DB
{
InterpolateDescription::InterpolateDescription(ActionsDAGPtr actions_, const Aliases & aliases)
: actions(actions_)
{
for (const auto & name_type : actions->getRequiredColumns())
{
if (const auto & p = aliases.find(name_type.name); p != aliases.end())
required_columns_map[p->second->getColumnName()] = name_type;
else
required_columns_map[name_type.name] = name_type;
}
for (const ColumnWithTypeAndName & column : actions->getResultColumns())
{
std::string name = column.name;
if (const auto & p = aliases.find(name); p != aliases.end())
name = p->second->getColumnName();
result_columns_set.insert(name);
result_columns_order.push_back(name);
}
}
}

33
src/Core/InterpolateDescription.h Normal file
View File

@ -0,0 +1,33 @@
#pragma once
#include <unordered_map>
#include <memory>
#include <cstddef>
#include <string>
#include <Core/Field.h>
#include <Core/SettingsEnums.h>
#include <Common/IntervalKind.h>
#include <Parsers/ASTOrderByElement.h>
#include <Parsers/ASTInterpolateElement.h>
#include <Functions/FunctionsMiscellaneous.h>
#include <Interpreters/Aliases.h>
namespace DB
{
/// Interpolate description
struct InterpolateDescription
{
explicit InterpolateDescription(ActionsDAGPtr actions, const Aliases & aliases);
ActionsDAGPtr actions;
std::unordered_map<std::string, NameAndTypePair> required_columns_map; /// input column name -> {alias, type}
std::unordered_set<std::string> result_columns_set; /// result block columns
std::vector<std::string> result_columns_order; /// result block columns order
};
using InterpolateDescriptionPtr = std::shared_ptr<InterpolateDescription>;
}

1
src/Core/SortDescription.h
View File

@ -7,6 +7,7 @@
#include <Core/Field.h>
#include <Core/SettingsEnums.h>
#include <Common/IntervalKind.h>
#include <DataTypes/IDataType.h>
class Collator;

29
src/DataTypes/DataTypeEnum.cpp
View File

@ -191,6 +191,33 @@ static void checkASTStructure(const ASTPtr & child)
ErrorCodes::UNEXPECTED_AST_STRUCTURE);
}
static void autoAssignNumberForEnum(const ASTPtr & arguments)
{
UInt64 literal_child_count = 0;
UInt64 func_child_count = 0;
ASTs assign_number_child;
assign_number_child.reserve(arguments->children.size());
for (const ASTPtr & child : arguments->children)
{
if (child->as<ASTLiteral>())
{
ASTPtr func = makeASTFunction("equals", child, std::make_shared<ASTLiteral>(++literal_child_count));
assign_number_child.emplace_back(func);
}
else
{
++func_child_count;
assign_number_child.emplace_back(child);
}
}
if (func_child_count > 0 && literal_child_count > 0)
throw Exception("ALL Elements of Enum data type must be of form: 'name' = number or 'name', where name is string literal and number is an integer",
ErrorCodes::UNEXPECTED_AST_STRUCTURE);
arguments->children = assign_number_child;
}
template <typename DataTypeEnum>
static DataTypePtr createExact(const ASTPtr & arguments)
{
@ -202,6 +229,7 @@ static DataTypePtr createExact(const ASTPtr & arguments)
using FieldType = typename DataTypeEnum::FieldType;
autoAssignNumberForEnum(arguments);
/// Children must be functions 'equals' with string literal as left argument and numeric literal as right argument.
for (const ASTPtr & child : arguments->children)
{
@ -236,6 +264,7 @@ static DataTypePtr create(const ASTPtr & arguments)
if (!arguments || arguments->children.empty())
throw Exception("Enum data type cannot be empty", ErrorCodes::EMPTY_DATA_PASSED);
autoAssignNumberForEnum(arguments);
/// Children must be functions 'equals' with string literal as left argument and numeric literal as right argument.
for (const ASTPtr & child : arguments->children)
{

5
src/DataTypes/ObjectUtils.cpp
View File

@ -139,11 +139,6 @@ void convertObjectsToTuples(Block & block, const NamesAndTypesList & extended_st
if (!isObject(column.type))
continue;
if (!isObject(column.type))
throw Exception(ErrorCodes::TYPE_MISMATCH,
"Type for column '{}' mismatch in columns list and in block. In list: {}, in block: {}",
column.name, column.type->getName(), column.type->getName());
const auto & column_object = assert_cast<const ColumnObject &>(*column.column);
const auto & subcolumns = column_object.getSubcolumns();

7
src/DataTypes/Serializations/SerializationObject.cpp
View File

@ -136,10 +136,17 @@ void SerializationObject<Parser>::deserializeTextImpl(IColumn & column, Reader &
reader(buf);
std::optional<ParseResult> result;
/// Treat empty string as an empty object
/// for better CAST from String to Object.
if (!buf.empty())
{
auto parser = parsers_pool.get([] { return new Parser; });
result = parser->parse(buf.data(), buf.size());
}
else
{
result = ParseResult{};
}
if (!result)
throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot parse object");

4
src/Databases/DatabaseAtomic.cpp
View File

@ -4,7 +4,7 @@
#include <IO/WriteHelpers.h>
#include <IO/ReadBufferFromFile.h>
#include <Parsers/formatAST.h>
#include <Common/renameat2.h>
#include <Common/atomicRename.h>
#include <Storages/StorageMaterializedView.h>
#include <Interpreters/Context.h>
#include <Interpreters/ExternalDictionariesLoader.h>
@ -158,7 +158,7 @@ void DatabaseAtomic::renameTable(ContextPtr local_context, const String & table_
return;
}
if (exchange && !supportsRenameat2())
if (exchange && !supportsAtomicRename())
throw Exception(ErrorCodes::NOT_IMPLEMENTED, "RENAME EXCHANGE is not supported");
auto & other_db = dynamic_cast<DatabaseAtomic &>(to_database);

1
src/Databases/DatabaseReplicatedWorker.cpp
View File

@ -319,7 +319,6 @@ bool DatabaseReplicatedDDLWorker::canRemoveQueueEntry(const String & entry_name,
void DatabaseReplicatedDDLWorker::initializeLogPointer(const String & processed_entry_name)
{
updateMaxDDLEntryID(processed_entry_name);
assert(max_id.load() == parse<UInt32>(getAndSetZooKeeper()->get(fs::path(database->replica_path) / "log_ptr")));
}
UInt32 DatabaseReplicatedDDLWorker::getLogPointer() const

11
src/Dictionaries/ExecutableDictionarySource.cpp
View File

@ -7,7 +7,6 @@
#include <base/logger_useful.h>
#include <Common/LocalDateTime.h>
#include <Common/filesystemHelpers.h>
#include <Common/ShellCommand.h>
#include <Processors/Sources/ShellCommandSource.h>
#include <Processors/Sources/SourceFromSingleChunk.h>
@ -15,12 +14,10 @@
#include <Interpreters/Context.h>
#include <IO/WriteHelpers.h>
#include <IO/ReadHelpers.h>
#include <Dictionaries/DictionarySourceFactory.h>
#include <Dictionaries/DictionarySourceHelpers.h>
#include <Dictionaries/DictionaryStructure.h>
#include <Dictionaries/registerDictionaries.h>
namespace DB
@ -51,12 +48,18 @@ namespace
command,
user_scripts_path);
if (!std::filesystem::exists(std::filesystem::path(script_path)))
if (!FS::exists(script_path))
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Executable file {} does not exist inside user scripts folder {}",
command,
user_scripts_path);
if (!FS::canExecute(script_path))
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Executable file {} is not executable inside user scripts folder {}",
command,
user_scripts_path);
command = std::move(script_path);
}

10
src/Dictionaries/ExecutablePoolDictionarySource.cpp
View File

@ -7,7 +7,6 @@
#include <base/logger_useful.h>
#include <Common/LocalDateTime.h>
#include <Common/filesystemHelpers.h>
#include <Common/ShellCommand.h>
#include <Processors/Formats/IOutputFormat.h>
#include <Processors/Sources/ShellCommandSource.h>
@ -20,7 +19,6 @@
#include <Dictionaries/DictionarySourceHelpers.h>
#include <Dictionaries/DictionaryStructure.h>
namespace DB
{
@ -113,12 +111,18 @@ Pipe ExecutablePoolDictionarySource::getStreamForBlock(const Block & block)
command,
user_scripts_path);
if (!std::filesystem::exists(std::filesystem::path(script_path)))
if (!FS::exists(script_path))
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Executable file {} does not exist inside user scripts folder {}",
command,
user_scripts_path);
if (!FS::canExecute(script_path))
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Executable file {} is not executable inside user scripts folder {}",
command,
user_scripts_path);
command = std::move(script_path);
}

55
src/Disks/AzureBlobStorage/DiskAzureBlobStorage.cpp
View File

@ -32,21 +32,6 @@ DiskAzureBlobStorageSettings::DiskAzureBlobStorageSettings(
thread_pool_size(thread_pool_size_) {}
class AzureBlobStoragePathKeeper : public RemoteFSPathKeeper
{
public:
/// RemoteFSPathKeeper constructed with a placeholder argument for chunk_limit, it is unused in this class
AzureBlobStoragePathKeeper() : RemoteFSPathKeeper(1000) {}
void addPath(const String & path) override
{
paths.push_back(path);
}
std::vector<String> paths;
};
DiskAzureBlobStorage::DiskAzureBlobStorage(
const String & name_,
DiskPtr metadata_disk_,
@ -71,8 +56,8 @@ std::unique_ptr<ReadBufferFromFileBase> DiskAzureBlobStorage::readFile(
LOG_TEST(log, "Read from file by path: {}", backQuote(metadata_disk->getPath() + path));
auto reader_impl = std::make_unique<ReadBufferFromAzureBlobStorageGather>(
path, blob_container_client, metadata, settings->max_single_read_retries,
settings->max_single_download_retries, read_settings);
blob_container_client, metadata.remote_fs_root_path, metadata.remote_fs_objects,
settings->max_single_read_retries, settings->max_single_download_retries, read_settings);
if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
{
@ -109,7 +94,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskAzureBlobStorage::writeFile(
readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_path, count] (Metadata & metadata) { metadata.addObject(blob_path, count); return true; });
};
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(create_metadata_callback), path);
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(buffer), std::move(create_metadata_callback), blob_path);
}
@ -150,36 +135,24 @@ bool DiskAzureBlobStorage::checkUniqueId(const String & id) const
}
void DiskAzureBlobStorage::removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper)
void DiskAzureBlobStorage::removeFromRemoteFS(const std::vector<String> & paths)
{
auto * paths_keeper = dynamic_cast<AzureBlobStoragePathKeeper *>(fs_paths_keeper.get());
if (paths_keeper)
for (const auto & path : paths)
{
for (const auto & path : paths_keeper->paths)
try
{
try
{
auto delete_info = blob_container_client->DeleteBlob(path);
if (!delete_info.Value.Deleted)
throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", path);
}
catch (const Azure::Storage::StorageException & e)
{
LOG_INFO(log, "Caught an error while deleting file {} : {}", path, e.Message);
throw;
}
auto delete_info = blob_container_client->DeleteBlob(path);
if (!delete_info.Value.Deleted)
throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Failed to delete file in AzureBlob Storage: {}", path);
}
catch (const Azure::Storage::StorageException & e)
{
LOG_INFO(log, "Caught an error while deleting file {} : {}", path, e.Message);
throw;
}
}
}
RemoteFSPathKeeperPtr DiskAzureBlobStorage::createFSPathKeeper() const
{
return std::make_shared<AzureBlobStoragePathKeeper>();
}
void DiskAzureBlobStorage::applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &)
{
auto new_settings = settings_getter(config, "storage_configuration.disks." + name, context);

4
src/Disks/AzureBlobStorage/DiskAzureBlobStorage.h
View File

@ -67,9 +67,7 @@ public:
bool checkUniqueId(const String & id) const override;
void removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) override;
RemoteFSPathKeeperPtr createFSPathKeeper() const override;
void removeFromRemoteFS(const std::vector<String> & paths) override;
void applyNewSettings(const Poco::Util::AbstractConfiguration & config, ContextPtr context, const String &, const DisksMap &) override;

2
src/Disks/DiskLocal.cpp
View File

@ -6,7 +6,7 @@
#include <Interpreters/Context.h>
#include <Common/filesystemHelpers.h>
#include <Common/quoteString.h>
#include <Common/renameat2.h>
#include <Common/atomicRename.h>
#include <Disks/IO/createReadBufferFromFileBase.h>
#include <fstream>

4
src/Disks/DiskWebServer.cpp
View File

@ -166,9 +166,9 @@ std::unique_ptr<ReadBufferFromFileBase> DiskWebServer::readFile(const String & p
remote_path = remote_path.string().substr(url.size());
RemoteMetadata meta(path, remote_path);
meta.remote_fs_objects.emplace_back(std::make_pair(remote_path, iter->second.size));
meta.remote_fs_objects.emplace_back(remote_path, iter->second.size);
auto web_impl = std::make_unique<ReadBufferFromWebServerGather>(path, url, meta, getContext(), read_settings);
auto web_impl = std::make_unique<ReadBufferFromWebServerGather>(url, meta.remote_fs_root_path, meta.remote_fs_objects, getContext(), read_settings);
if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
{

68
src/Disks/HDFS/DiskHDFS.cpp
View File

@ -30,35 +30,6 @@ namespace ErrorCodes
}
class HDFSPathKeeper : public RemoteFSPathKeeper
{
public:
using Chunk = std::vector<std::string>;
using Chunks = std::list<Chunk>;
explicit HDFSPathKeeper(size_t chunk_limit_) : RemoteFSPathKeeper(chunk_limit_) {}
void addPath(const String & path) override
{
if (chunks.empty() || chunks.back().size() >= chunk_limit)
{
chunks.push_back(Chunks::value_type());
chunks.back().reserve(chunk_limit);
}
chunks.back().push_back(path.data());
}
void removePaths(Fn<void(Chunk &&)> auto && remove_chunk_func)
{
for (auto & chunk : chunks)
remove_chunk_func(std::move(chunk));
}
private:
Chunks chunks;
};
DiskHDFS::DiskHDFS(
const String & disk_name_,
const String & hdfs_root_path_,
@ -82,7 +53,7 @@ std::unique_ptr<ReadBufferFromFileBase> DiskHDFS::readFile(const String & path,
"Read from file by path: {}. Existing HDFS objects: {}",
backQuote(metadata_disk->getPath() + path), metadata.remote_fs_objects.size());
auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(path, config, remote_fs_root_path, metadata, read_settings);
auto hdfs_impl = std::make_unique<ReadBufferFromHDFSGather>(config, remote_fs_root_path, remote_fs_root_path, metadata.remote_fs_objects, read_settings);
auto buf = std::make_unique<ReadIndirectBufferFromRemoteFS>(std::move(hdfs_impl));
return std::make_unique<SeekAvoidingReadBuffer>(std::move(buf), settings->min_bytes_for_seek);
}
@ -91,8 +62,8 @@ std::unique_ptr<ReadBufferFromFileBase> DiskHDFS::readFile(const String & path,
std::unique_ptr<WriteBufferFromFileBase> DiskHDFS::writeFile(const String & path, size_t buf_size, WriteMode mode, const WriteSettings &)
{
/// Path to store new HDFS object.
auto file_name = getRandomName();
auto hdfs_path = remote_fs_root_path + file_name;
std::string file_name = getRandomName();
std::string hdfs_path = fs::path(remote_fs_root_path) / file_name;
LOG_TRACE(log, "{} to file by path: {}. HDFS path: {}", mode == WriteMode::Rewrite ? "Write" : "Append",
backQuote(metadata_disk->getPath() + path), hdfs_path);
@ -106,33 +77,20 @@ std::unique_ptr<WriteBufferFromFileBase> DiskHDFS::writeFile(const String & path
readOrCreateUpdateAndStoreMetadata(path, mode, false, [file_name, count] (Metadata & metadata) { metadata.addObject(file_name, count); return true; });
};
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(create_metadata_callback), path);
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(hdfs_buffer), std::move(create_metadata_callback), hdfs_path);
}
RemoteFSPathKeeperPtr DiskHDFS::createFSPathKeeper() const
void DiskHDFS::removeFromRemoteFS(const std::vector<String> & paths)
{
return std::make_shared<HDFSPathKeeper>(settings->objects_chunk_size_to_delete);
}
for (const auto & hdfs_path : paths)
{
const size_t begin_of_path = hdfs_path.find('/', hdfs_path.find("//") + 2);
void DiskHDFS::removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper)
{
auto * hdfs_paths_keeper = dynamic_cast<HDFSPathKeeper *>(fs_paths_keeper.get());
if (hdfs_paths_keeper)
hdfs_paths_keeper->removePaths([&](std::vector<std::string> && chunk)
{
for (const auto & hdfs_object_path : chunk)
{
const String & hdfs_path = hdfs_object_path;
const size_t begin_of_path = hdfs_path.find('/', hdfs_path.find("//") + 2);
/// Add path from root to file name
int res = hdfsDelete(hdfs_fs.get(), hdfs_path.substr(begin_of_path).c_str(), 0);
if (res == -1)
throw Exception(ErrorCodes::LOGICAL_ERROR, "HDFSDelete failed with path: " + hdfs_path);
}
});
/// Add path from root to file name
int res = hdfsDelete(hdfs_fs.get(), hdfs_path.substr(begin_of_path).c_str(), 0);
if (res == -1)
throw Exception(ErrorCodes::LOGICAL_ERROR, "HDFSDelete failed with path: " + hdfs_path);
}
}
bool DiskHDFS::checkUniqueId(const String & hdfs_uri) const

4
src/Disks/HDFS/DiskHDFS.h
View File

@ -62,9 +62,7 @@ public:
std::unique_ptr<WriteBufferFromFileBase> writeFile(const String & path, size_t buf_size, WriteMode mode, const WriteSettings & settings) override;
void removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) override;
RemoteFSPathKeeperPtr createFSPathKeeper() const override;
void removeFromRemoteFS(const std::vector<String> & paths) override;
/// Check file exists and ClickHouse has an access to it
/// Overrode in remote disk

44
src/Disks/IDiskRemote.cpp
View File

@ -122,7 +122,8 @@ void IDiskRemote::Metadata::load()
remote_fs_object_path = remote_fs_object_path.substr(remote_fs_root_path.size());
}
assertChar('\n', *buf);
remote_fs_objects[i] = {remote_fs_object_path, remote_fs_object_size};
remote_fs_objects[i].relative_path = remote_fs_object_path;
remote_fs_objects[i].bytes_size = remote_fs_object_size;
}
readIntText(ref_count, *buf);
@ -269,7 +270,7 @@ std::unordered_map<String, String> IDiskRemote::getSerializedMetadata(const std:
return metadatas;
}
void IDiskRemote::removeMetadata(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper)
void IDiskRemote::removeMetadata(const String & path, std::vector<std::string> & paths_to_remove)
{
LOG_TRACE(log, "Remove file by path: {}", backQuote(metadata_disk->getPath() + path));
@ -281,13 +282,13 @@ void IDiskRemote::removeMetadata(const String & path, RemoteFSPathKeeperPtr fs_p
try
{
auto metadata_updater = [fs_paths_keeper, this] (Metadata & metadata)
auto metadata_updater = [&paths_to_remove, this] (Metadata & metadata)
{
if (metadata.ref_count == 0)
{
for (const auto & [remote_fs_object_path, _] : metadata.remote_fs_objects)
{
fs_paths_keeper->addPath(remote_fs_root_path + remote_fs_object_path);
paths_to_remove.push_back(remote_fs_root_path + remote_fs_object_path);
if (cache)
{
@ -326,18 +327,18 @@ void IDiskRemote::removeMetadata(const String & path, RemoteFSPathKeeperPtr fs_p
}
void IDiskRemote::removeMetadataRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper)
void IDiskRemote::removeMetadataRecursive(const String & path, std::vector<String> & paths_to_remove)
{
checkStackSize(); /// This is needed to prevent stack overflow in case of cyclic symlinks.
if (metadata_disk->isFile(path))
{
removeMetadata(path, fs_paths_keeper);
removeMetadata(path, paths_to_remove);
}
else
{
for (auto it = iterateDirectory(path); it->isValid(); it->next())
removeMetadataRecursive(it->path(), fs_paths_keeper);
removeMetadataRecursive(it->path(), paths_to_remove);
metadata_disk->removeDirectory(path);
}
@ -479,50 +480,47 @@ void IDiskRemote::replaceFile(const String & from_path, const String & to_path)
moveFile(from_path, to_path);
}
void IDiskRemote::removeSharedFile(const String & path, bool delete_metadata_only)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
removeMetadata(path, fs_paths_keeper);
std::vector<String> paths_to_remove;
removeMetadata(path, paths_to_remove);
if (!delete_metadata_only)
removeFromRemoteFS(fs_paths_keeper);
removeFromRemoteFS(paths_to_remove);
}
void IDiskRemote::removeSharedFileIfExists(const String & path, bool delete_metadata_only)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
std::vector<String> paths_to_remove;
if (metadata_disk->exists(path))
{
removeMetadata(path, fs_paths_keeper);
removeMetadata(path, paths_to_remove);
if (!delete_metadata_only)
removeFromRemoteFS(fs_paths_keeper);
removeFromRemoteFS(paths_to_remove);
}
}
void IDiskRemote::removeSharedFiles(const RemoveBatchRequest & files, bool delete_metadata_only)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
std::vector<String> paths_to_remove;
for (const auto & file : files)
{
bool skip = file.if_exists && !metadata_disk->exists(file.path);
if (!skip)
removeMetadata(file.path, fs_paths_keeper);
removeMetadata(file.path, paths_to_remove);
}
if (!delete_metadata_only)
removeFromRemoteFS(fs_paths_keeper);
removeFromRemoteFS(paths_to_remove);
}
void IDiskRemote::removeSharedRecursive(const String & path, bool delete_metadata_only)
{
RemoteFSPathKeeperPtr fs_paths_keeper = createFSPathKeeper();
removeMetadataRecursive(path, fs_paths_keeper);
std::vector<String> paths_to_remove;
removeMetadataRecursive(path, paths_to_remove);
if (!delete_metadata_only)
removeFromRemoteFS(fs_paths_keeper);
removeFromRemoteFS(paths_to_remove);
}
@ -638,7 +636,7 @@ String IDiskRemote::getUniqueId(const String & path) const
auto metadata = readMetadata(path);
String id;
if (!metadata.remote_fs_objects.empty())
id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].first;
id = metadata.remote_fs_root_path + metadata.remote_fs_objects[0].relative_path;
return id;
}

37
src/Disks/IDiskRemote.h
View File

@ -13,7 +13,6 @@
#include <Common/ThreadPool.h>
#include <filesystem>
namespace CurrentMetrics
{
extern const Metric DiskSpaceReservedForMerge;
@ -22,23 +21,23 @@ namespace CurrentMetrics
namespace DB
{
/// Helper class to collect paths into chunks of maximum size.
/// For s3 it is Aws::vector<ObjectIdentifier>, for hdfs it is std::vector<std::string>.
class RemoteFSPathKeeper
/// Path to blob with it's size
struct BlobPathWithSize
{
public:
explicit RemoteFSPathKeeper(size_t chunk_limit_) : chunk_limit(chunk_limit_) {}
std::string relative_path;
uint64_t bytes_size;
virtual ~RemoteFSPathKeeper() = default;
BlobPathWithSize() = default;
BlobPathWithSize(const BlobPathWithSize & other) = default;
virtual void addPath(const String & path) = 0;
protected:
size_t chunk_limit;
BlobPathWithSize(const std::string & relative_path_, uint64_t bytes_size_)
: relative_path(relative_path_)
, bytes_size(bytes_size_)
{}
};
using RemoteFSPathKeeperPtr = std::shared_ptr<RemoteFSPathKeeper>;
/// List of blobs with their sizes
using BlobsPathToSize = std::vector<BlobPathWithSize>;
class IAsynchronousReader;
using AsynchronousReaderPtr = std::shared_ptr<IAsynchronousReader>;
@ -148,9 +147,7 @@ public:
bool checkUniqueId(const String & id) const override = 0;
virtual void removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper) = 0;
virtual RemoteFSPathKeeperPtr createFSPathKeeper() const = 0;
virtual void removeFromRemoteFS(const std::vector<String> & paths) = 0;
static AsynchronousReaderPtr getThreadPoolReader();
static ThreadPool & getThreadPoolWriter();
@ -173,9 +170,9 @@ protected:
FileCachePtr cache;
private:
void removeMetadata(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper);
void removeMetadata(const String & path, std::vector<String> & paths_to_remove);
void removeMetadataRecursive(const String & path, RemoteFSPathKeeperPtr fs_paths_keeper);
void removeMetadataRecursive(const String & path, std::vector<String> & paths_to_remove);
bool tryReserve(UInt64 bytes);
@ -191,10 +188,8 @@ using RemoteDiskPtr = std::shared_ptr<IDiskRemote>;
/// Minimum info, required to be passed to ReadIndirectBufferFromRemoteFS<T>
struct RemoteMetadata
{
using PathAndSize = std::pair<String, size_t>;
/// Remote FS objects paths and their sizes.
std::vector<PathAndSize> remote_fs_objects;
std::vector<BlobPathWithSize> remote_fs_objects;
/// URI
const String & remote_fs_root_path;

26
src/Disks/IO/CachedReadBufferFromRemoteFS.cpp
View File

@ -46,7 +46,15 @@ CachedReadBufferFromRemoteFS::CachedReadBufferFromRemoteFS(
void CachedReadBufferFromRemoteFS::initialize(size_t offset, size_t size)
{
file_segments_holder.emplace(cache->getOrSet(cache_key, offset, size));
if (settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache)
{
file_segments_holder.emplace(cache->get(cache_key, offset, size));
}
else
{
file_segments_holder.emplace(cache->getOrSet(cache_key, offset, size));
}
/**
* Segments in returned list are ordered in ascending order and represent a full contiguous
@ -326,6 +334,10 @@ SeekableReadBufferPtr CachedReadBufferFromRemoteFS::getImplementationBuffer(File
#endif
size_t seek_offset = file_offset_of_buffer_end - range.left;
if (file_offset_of_buffer_end < range.left)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Invariant failed. Expected {} > {} (current offset > file segment's start offset)", file_offset_of_buffer_end, range.left);
read_buffer_for_file_segment->seek(seek_offset, SEEK_SET);
break;
@ -577,6 +589,8 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
{
last_caller_id = FileSegment::getCallerId();
assertCorrectness();
if (!initialized)
initialize(file_offset_of_buffer_end, getTotalSizeToRead());
@ -597,8 +611,8 @@ bool CachedReadBufferFromRemoteFS::nextImplStep()
{
try
{
bool file_segment_already_completed = !file_segment->isDownloader();
if (!file_segment_already_completed)
bool need_complete_file_segment = file_segment->isDownloader();
if (need_complete_file_segment)
file_segment->completeBatchAndResetDownloader();
}
catch (...)
@ -820,6 +834,12 @@ std::optional<size_t> CachedReadBufferFromRemoteFS::getLastNonDownloadedOffset()
return std::nullopt;
}
void CachedReadBufferFromRemoteFS::assertCorrectness() const
{
if (IFileCache::isReadOnly() && !settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache)
throw Exception(ErrorCodes::LOGICAL_ERROR, "Cache usage is not allowed");
}
String CachedReadBufferFromRemoteFS::getInfoForLog()
{
String implementation_buffer_read_range_str;

2
src/Disks/IO/CachedReadBufferFromRemoteFS.h
View File

@ -50,6 +50,8 @@ private:
bool nextImplStep();
void assertCorrectness() const;
enum class ReadType
{
CACHED,

31
src/Disks/IO/ReadBufferFromRemoteFSGather.cpp
View File

@ -38,12 +38,14 @@ SeekableReadBufferPtr ReadBufferFromS3Gather::createImplementationBuffer(const S
current_path = path;
auto cache = settings.remote_fs_cache;
bool with_cache = cache && settings.enable_filesystem_cache;
bool with_cache = cache
&& settings.enable_filesystem_cache
&& (!IFileCache::isReadOnly() || settings.read_from_filesystem_cache_if_exists_otherwise_bypass_cache);
auto remote_file_reader_creator = [=, this]()
{
return std::make_unique<ReadBufferFromS3>(
client_ptr, bucket, fs::path(metadata.remote_fs_root_path) / path, max_single_read_retries,
client_ptr, bucket, fs::path(common_path_prefix) / path, max_single_read_retries,
settings, /* use_external_buffer */true, /* offset */ 0, read_until_position, /* restricted_seek */true);
};
@ -83,11 +85,14 @@ SeekableReadBufferPtr ReadBufferFromHDFSGather::createImplementationBuffer(const
#endif
ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(const RemoteMetadata & metadata_, const ReadSettings & settings_, const String & path_)
ReadBufferFromRemoteFSGather::ReadBufferFromRemoteFSGather(
const std::string & common_path_prefix_,
const BlobsPathToSize & blobs_to_read_,
const ReadSettings & settings_)
: ReadBuffer(nullptr, 0)
, metadata(metadata_)
, common_path_prefix(common_path_prefix_)
, blobs_to_read(blobs_to_read_)
, settings(settings_)
, canonical_path(path_)
, log(&Poco::Logger::get("ReadBufferFromRemoteFSGather"))
{
}
@ -119,9 +124,9 @@ void ReadBufferFromRemoteFSGather::initialize()
{
/// One clickhouse file can be split into multiple files in remote fs.
auto current_buf_offset = file_offset_of_buffer_end;
for (size_t i = 0; i < metadata.remote_fs_objects.size(); ++i)
for (size_t i = 0; i < blobs_to_read.size(); ++i)
{
const auto & [file_path, size] = metadata.remote_fs_objects[i];
const auto & [file_path, size] = blobs_to_read[i];
if (size > current_buf_offset)
{
@ -138,7 +143,7 @@ void ReadBufferFromRemoteFSGather::initialize()
current_buf_offset -= size;
}
current_buf_idx = metadata.remote_fs_objects.size();
current_buf_idx = blobs_to_read.size();
current_buf = nullptr;
}
@ -168,12 +173,12 @@ bool ReadBufferFromRemoteFSGather::nextImpl()
bool ReadBufferFromRemoteFSGather::moveToNextBuffer()
{
/// If there is no available buffers - nothing to read.
if (current_buf_idx + 1 >= metadata.remote_fs_objects.size())
if (current_buf_idx + 1 >= blobs_to_read.size())
return false;
++current_buf_idx;
const auto & [path, size] = metadata.remote_fs_objects[current_buf_idx];
const auto & [path, size] = blobs_to_read[current_buf_idx];
current_buf = createImplementationBuffer(path, size);
return true;
@ -202,7 +207,7 @@ bool ReadBufferFromRemoteFSGather::readImpl()
if (!result)
result = current_buf->next();
if (metadata.remote_fs_objects.size() == 1)
if (blobs_to_read.size() == 1)
{
file_offset_of_buffer_end = current_buf->getFileOffsetOfBufferEnd();
}
@ -255,8 +260,8 @@ String ReadBufferFromRemoteFSGather::getFileName() const
size_t ReadBufferFromRemoteFSGather::getFileSize() const
{
size_t size = 0;
for (const auto & object : metadata.remote_fs_objects)
size += object.second;
for (const auto & object : blobs_to_read)
size += object.bytes_size;
return size;
}

36
src/Disks/IO/ReadBufferFromRemoteFSGather.h
View File

@ -26,9 +26,9 @@ friend class ReadIndirectBufferFromRemoteFS;
public:
ReadBufferFromRemoteFSGather(
const RemoteMetadata & metadata_,
const ReadSettings & settings_,
const String & path_);
const std::string & common_path_prefix_,
const BlobsPathToSize & blobs_to_read_,
const ReadSettings & settings_);
String getFileName() const;
@ -57,7 +57,9 @@ public:
protected:
virtual SeekableReadBufferPtr createImplementationBuffer(const String & path, size_t file_size) = 0;
RemoteMetadata metadata;
std::string common_path_prefix;
BlobsPathToSize blobs_to_read;
ReadSettings settings;
@ -89,8 +91,6 @@ private:
*/
size_t bytes_to_ignore = 0;
String canonical_path;
Poco::Logger * log;
};
@ -101,13 +101,13 @@ class ReadBufferFromS3Gather final : public ReadBufferFromRemoteFSGather
{
public:
ReadBufferFromS3Gather(
const String & path_,
std::shared_ptr<Aws::S3::S3Client> client_ptr_,
const String & bucket_,
IDiskRemote::Metadata metadata_,
const std::string & common_path_prefix_,
const BlobsPathToSize & blobs_to_read_,
size_t max_single_read_retries_,
const ReadSettings & settings_)
: ReadBufferFromRemoteFSGather(metadata_, settings_, path_)
: ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_)
, client_ptr(std::move(client_ptr_))
, bucket(bucket_)
, max_single_read_retries(max_single_read_retries_)
@ -130,13 +130,13 @@ class ReadBufferFromAzureBlobStorageGather final : public ReadBufferFromRemoteFS
{
public:
ReadBufferFromAzureBlobStorageGather(
const String & path_,
std::shared_ptr<Azure::Storage::Blobs::BlobContainerClient> blob_container_client_,
IDiskRemote::Metadata metadata_,
const std::string & common_path_prefix_,
const BlobsPathToSize & blobs_to_read_,
size_t max_single_read_retries_,
size_t max_single_download_retries_,
const ReadSettings & settings_)
: ReadBufferFromRemoteFSGather(metadata_, settings_, path_)
: ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_)
, blob_container_client(blob_container_client_)
, max_single_read_retries(max_single_read_retries_)
, max_single_download_retries(max_single_download_retries_)
@ -157,12 +157,12 @@ class ReadBufferFromWebServerGather final : public ReadBufferFromRemoteFSGather
{
public:
ReadBufferFromWebServerGather(
const String & path_,
const String & uri_,
RemoteMetadata metadata_,
const std::string & common_path_prefix_,
const BlobsPathToSize & blobs_to_read_,
ContextPtr context_,
const ReadSettings & settings_)
: ReadBufferFromRemoteFSGather(metadata_, settings_, path_)
: ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_)
, uri(uri_)
, context(context_)
{
@ -182,12 +182,12 @@ class ReadBufferFromHDFSGather final : public ReadBufferFromRemoteFSGather
{
public:
ReadBufferFromHDFSGather(
const String & path_,
const Poco::Util::AbstractConfiguration & config_,
const String & hdfs_uri_,
IDiskRemote::Metadata metadata_,
const std::string & common_path_prefix_,
const BlobsPathToSize & blobs_to_read_,
const ReadSettings & settings_)
: ReadBufferFromRemoteFSGather(metadata_, settings_, path_)
: ReadBufferFromRemoteFSGather(common_path_prefix_, blobs_to_read_, settings_)
, config(config_)
{
const size_t begin_of_path = hdfs_uri_.find('/', hdfs_uri_.find("//") + 2);

4
src/Disks/IO/WriteIndirectBufferFromRemoteFS.cpp
View File

@ -12,10 +12,10 @@ namespace DB
WriteIndirectBufferFromRemoteFS::WriteIndirectBufferFromRemoteFS(
std::unique_ptr<WriteBuffer> impl_,
CreateMetadataCallback && create_callback_,
const String & metadata_file_path_)
const String & remote_path_)
: WriteBufferFromFileDecorator(std::move(impl_))
, create_metadata_callback(std::move(create_callback_))
, metadata_file_path(metadata_file_path_)
, remote_path(remote_path_)
{
}

6
src/Disks/IO/WriteIndirectBufferFromRemoteFS.h
View File

@ -18,17 +18,17 @@ public:
WriteIndirectBufferFromRemoteFS(
std::unique_ptr<WriteBuffer> impl_,
CreateMetadataCallback && create_callback_,
const String & metadata_file_path_);
const String & remote_path_);
~WriteIndirectBufferFromRemoteFS() override;
String getFileName() const override { return metadata_file_path; }
String getFileName() const override { return remote_path; }
private:
void finalizeImpl() override;
CreateMetadataCallback create_metadata_callback;
String metadata_file_path;
String remote_path;
};
}

99
src/Disks/S3/DiskS3.cpp
View File

@ -60,52 +60,6 @@ namespace ErrorCodes
extern const int LOGICAL_ERROR;
}
/// Helper class to collect keys into chunks of maximum size (to prepare batch requests to AWS API)
/// see https://docs.aws.amazon.com/AmazonS3/latest/API/API_DeleteObjects.html
class S3PathKeeper : public RemoteFSPathKeeper
{
public:
using Chunk = Aws::Vector<Aws::S3::Model::ObjectIdentifier>;
using Chunks = std::list<Chunk>;
explicit S3PathKeeper(size_t chunk_limit_) : RemoteFSPathKeeper(chunk_limit_) {}
void addPath(const String & path) override
{
if (chunks.empty() || chunks.back().size() >= chunk_limit)
{
/// add one more chunk
chunks.push_back(Chunks::value_type());
chunks.back().reserve(chunk_limit);
}
Aws::S3::Model::ObjectIdentifier obj;
obj.SetKey(path);
chunks.back().push_back(obj);
}
void removePaths(Fn<void(Chunk &&)> auto && remove_chunk_func)
{
for (auto & chunk : chunks)
remove_chunk_func(std::move(chunk));
}
static String getChunkKeys(const Chunk & chunk)
{
String res;
for (const auto & obj : chunk)
{
const auto & key = obj.GetKey();
if (!res.empty())
res.append(", ");
res.append(key.c_str(), key.size());
}
return res;
}
private:
Chunks chunks;
};
template <typename Result, typename Error>
void throwIfError(Aws::Utils::Outcome<Result, Error> & response)
{
@ -168,31 +122,36 @@ DiskS3::DiskS3(
{
}
RemoteFSPathKeeperPtr DiskS3::createFSPathKeeper() const
void DiskS3::removeFromRemoteFS(const std::vector<String> & paths)
{
auto settings = current_settings.get();
return std::make_shared<S3PathKeeper>(settings->objects_chunk_size_to_delete);
}
void DiskS3::removeFromRemoteFS(RemoteFSPathKeeperPtr fs_paths_keeper)
{
auto settings = current_settings.get();
auto * s3_paths_keeper = dynamic_cast<S3PathKeeper *>(fs_paths_keeper.get());
if (s3_paths_keeper)
s3_paths_keeper->removePaths([&](S3PathKeeper::Chunk && chunk)
size_t chunk_size_limit = settings->objects_chunk_size_to_delete;
size_t current_position = 0;
while (current_position < paths.size())
{
std::vector<Aws::S3::Model::ObjectIdentifier> current_chunk;
String keys;
for (; current_position < paths.size() && current_chunk.size() < chunk_size_limit; ++current_position)
{
String keys = S3PathKeeper::getChunkKeys(chunk);
LOG_TRACE(log, "Remove AWS keys {}", keys);
Aws::S3::Model::Delete delkeys;
delkeys.SetObjects(chunk);
Aws::S3::Model::DeleteObjectsRequest request;
request.SetBucket(bucket);
request.SetDelete(delkeys);
auto outcome = settings->client->DeleteObjects(request);
// Do not throw here, continue deleting other chunks
logIfError(outcome, [&](){return "Can't remove AWS keys: " + keys;});
});
Aws::S3::Model::ObjectIdentifier obj;
obj.SetKey(paths[current_position]);
current_chunk.push_back(obj);
if (!keys.empty())
keys += ", ";
keys += paths[current_position];
}
LOG_TRACE(log, "Remove AWS keys {}", keys);
Aws::S3::Model::Delete delkeys;
delkeys.SetObjects(current_chunk);
Aws::S3::Model::DeleteObjectsRequest request;
request.SetBucket(bucket);
request.SetDelete(delkeys);
auto outcome = settings->client->DeleteObjects(request);
logIfError(outcome, [&](){return "Can't remove AWS keys: " + keys;});
}
}
void DiskS3::moveFile(const String & from_path, const String & to_path)
@ -237,7 +196,7 @@ std::unique_ptr<ReadBufferFromFileBase> DiskS3::readFile(const String & path, co
}
auto s3_impl = std::make_unique<ReadBufferFromS3Gather>(
path, settings->client, bucket, metadata,
settings->client, bucket, metadata.remote_fs_root_path, metadata.remote_fs_objects,
settings->s3_max_single_read_retries, disk_read_settings);
if (read_settings.remote_fs_method == RemoteFSReadMethod::threadpool)
@ -280,7 +239,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskS3::writeFile(const String & path,
auto s3_buffer = std::make_unique<WriteBufferFromS3>(
settings->client,
bucket,
remote_fs_root_path + blob_name,
fs::path(remote_fs_root_path) / blob_name,
settings->s3_min_upload_part_size,
settings->s3_upload_part_size_multiply_factor,
settings->s3_upload_part_size_multiply_parts_count_threshold,
@ -293,7 +252,7 @@ std::unique_ptr<WriteBufferFromFileBase> DiskS3::writeFile(const String & path,
readOrCreateUpdateAndStoreMetadata(path, mode, false, [blob_name, count] (Metadata & metadata) { metadata.addObject(blob_name, count); return true; });
};
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(s3_buffer), std::move(create_metadata_callback), path);
return std::make_unique<WriteIndirectBufferFromRemoteFS>(std::move(s3_buffer), std::move(create_metadata_callback), fs::path(remote_fs_root_path) / blob_name);
}
void DiskS3::createHardLink(const String & src_path, const String & dst_path)

4
src/Disks/S3/DiskS3.h
View File

@ -91,9 +91,7 @@ public:
WriteMode mode,
const WriteSettings & settings) override;
void removeFromRemoteFS(RemoteFSPathKeeperPtr keeper) override;
RemoteFSPathKeeperPtr createFSPathKeeper() const override;
void removeFromRemoteFS(const std::vector<String> & paths) override;
void moveFile(const String & from_path, const String & to_path, bool send_metadata);
void moveFile(const String & from_path, const String & to_path) override;

7
src/Formats/JSONEachRowUtils.cpp
View File

@ -255,6 +255,10 @@ struct JSONEachRowFieldsExtractor
std::vector<Element> extract(const Element & element)
{
/// {..., "<column_name>" : <value>, ...}
if (!element.isObject())
throw Exception(ErrorCodes::INCORRECT_DATA, "Root JSON value is not an object");
auto object = element.getObject();
std::vector<Element> fields;
fields.reserve(object.size());
@ -287,6 +291,9 @@ struct JSONCompactEachRowFieldsExtractor
std::vector<Element> extract(const Element & element)
{
/// [..., <value>, ...]
if (!element.isArray())
throw Exception(ErrorCodes::INCORRECT_DATA, "Root JSON value is not an array");
auto array = element.getArray();
std::vector<Element> fields;
fields.reserve(array.size());

3
src/Functions/h3GetRes0Indexes.cpp
View File

@ -50,11 +50,8 @@ public:
res0_indexes.resize(cell_count);
getRes0Cells(res0_indexes.data());
auto res = ColumnArray::create(ColumnUInt64::create());
Array res_indexes;
res_indexes.insert(res_indexes.end(), res0_indexes.begin(), res0_indexes.end());
res->insert(res_indexes);
return result_type->createColumnConst(input_rows_count, res_indexes);
}

363
src/Functions/makeDate.cpp
View File

@ -2,7 +2,11 @@
#include <Functions/FunctionFactory.h>
#include <DataTypes/DataTypeDate.h>
#include <DataTypes/DataTypeDate32.h>
#include <DataTypes/DataTypeDateTime.h>
#include <DataTypes/DataTypeDateTime64.h>
#include <DataTypes/DataTypesNumber.h>
#include <Columns/ColumnConst.h>
#include <Columns/ColumnDecimal.h>
#include <Columns/ColumnsNumber.h>
#include <Interpreters/castColumn.h>
@ -17,20 +21,69 @@ namespace ErrorCodes
{
extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
extern const int ILLEGAL_TYPE_OF_ARGUMENT;
extern const int ARGUMENT_OUT_OF_BOUND;
}
namespace
{
// A helper function to simplify comparisons of valid YYYY-MM-DD values for <,>,=
/// A helper function to simplify comparisons of valid YYYY-MM-DD values for <,>,=
inline constexpr Int64 YearMonthDayToSingleInt(Int64 year, Int64 month, Int64 day)
{
return year * 512 + month * 32 + day;
}
// Common implementation for makeDate, makeDate32
/// Common logic to handle numeric arguments like year, month, day, hour, minute, second
class FunctionWithNumericParamsBase : public IFunction
{
public:
bool isInjective(const ColumnsWithTypeAndName &) const override
{
return false; /// invalid argument values and timestamps that are out of supported range are converted into a default value
}
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
bool useDefaultImplementationForNulls() const override { return true; }
bool useDefaultImplementationForConstants() const override { return true; }
protected:
template <class AgrumentNames>
void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, const AgrumentNames & argument_names, const size_t optional_argument_count) const
{
if (arguments.size() < argument_names.size() || arguments.size() > argument_names.size() + optional_argument_count)
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Function {} requires {} to {} arguments, but {} given",
getName(), argument_names.size(), argument_names.size() + optional_argument_count, arguments.size());
for (size_t i = 0; i < argument_names.size(); ++i)
{
DataTypePtr argument_type = arguments[i].type;
if (!isNumber(argument_type))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Argument '{}' for function {} must be number", std::string(argument_names[i]), getName());
}
}
template <class AgrumentNames>
void convertRequiredArguments(const ColumnsWithTypeAndName & arguments, const AgrumentNames & argument_names, Columns & converted_arguments) const
{
const DataTypePtr converted_argument_type = std::make_shared<DataTypeFloat32>();
converted_arguments.clear();
converted_arguments.reserve(arguments.size());
for (size_t i = 0; i < argument_names.size(); ++i)
{
ColumnPtr argument_column = castColumn(arguments[i], converted_argument_type);
argument_column = argument_column->convertToFullColumnIfConst();
converted_arguments.push_back(argument_column);
}
}
};
/// Common implementation for makeDate, makeDate32
template <typename Traits>
class FunctionMakeDate : public IFunction
class FunctionMakeDate : public FunctionWithNumericParamsBase
{
private:
static constexpr std::array<const char*, 3> argument_names = {"year", "month", "day"};
@ -46,45 +99,17 @@ public:
size_t getNumberOfArguments() const override { return argument_names.size(); }
bool isInjective(const ColumnsWithTypeAndName &) const override
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
return false; // {year,month,day} that are out of supported range are converted into a default value
}
bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return true; }
bool useDefaultImplementationForNulls() const override { return true; }
bool useDefaultImplementationForConstants() const override { return true; }
DataTypePtr getReturnTypeImpl(const DataTypes & arguments) const override
{
if (arguments.size() != argument_names.size())
throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH,
"Function {} requires 3 arguments, but {} given", getName(), arguments.size());
for (size_t i = 0; i < argument_names.size(); ++i)
{
DataTypePtr argument_type = arguments[i];
if (!isNumber(argument_type))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Argument '{}' for function {} must be number", std::string(argument_names[i]), getName());
}
checkRequiredArguments(arguments, argument_names, 0);
return std::make_shared<typename Traits::ReturnDataType>();
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
const DataTypePtr converted_argument_type = std::make_shared<DataTypeFloat32>();
Columns converted_arguments;
converted_arguments.reserve(arguments.size());
for (const auto & argument : arguments)
{
ColumnPtr argument_column = castColumn(argument, converted_argument_type);
argument_column = argument_column->convertToFullColumnIfConst();
converted_arguments.push_back(argument_column);
}
convertRequiredArguments(arguments, argument_names, converted_arguments);
auto res_column = Traits::ReturnColumnType::create(input_rows_count);
auto & result_data = res_column->getData();
@ -119,7 +144,7 @@ public:
}
};
// makeDate(year, month, day)
/// makeDate(year, month, day)
struct MakeDateTraits
{
static constexpr auto name = "makeDate";
@ -128,11 +153,11 @@ struct MakeDateTraits
static constexpr auto MIN_YEAR = 1970;
static constexpr auto MAX_YEAR = 2149;
// This date has the maximum day number that fits in 16-bit uint
/// This date has the maximum day number that fits in 16-bit uint
static constexpr auto MAX_DATE = YearMonthDayToSingleInt(MAX_YEAR, 6, 6);
};
// makeDate32(year, month, day)
/// makeDate32(year, month, day)
struct MakeDate32Traits
{
static constexpr auto name = "makeDate32";
@ -144,12 +169,276 @@ struct MakeDate32Traits
static constexpr auto MAX_DATE = YearMonthDayToSingleInt(MAX_YEAR, 11, 11);
};
/// Common implementation for makeDateTime, makeDateTime64
class FunctionMakeDateTimeBase : public FunctionWithNumericParamsBase
{
protected:
static constexpr std::array<const char*, 6> argument_names = {"year", "month", "day", "hour", "minute", "second"};
public:
bool isVariadic() const override { return true; }
size_t getNumberOfArguments() const override { return 0; }
protected:
void checkRequiredArguments(const ColumnsWithTypeAndName & arguments, const size_t optional_argument_count) const
{
FunctionWithNumericParamsBase::checkRequiredArguments(arguments, argument_names, optional_argument_count);
}
void convertRequiredArguments(const ColumnsWithTypeAndName & arguments, Columns & converted_arguments) const
{
FunctionWithNumericParamsBase::convertRequiredArguments(arguments, argument_names, converted_arguments);
}
template <typename T>
static Int64 dateTime(T year, T month, T day_of_month, T hour, T minute, T second, const DateLUTImpl & lut)
{
/// Note that hour, minute and second are checked against 99 to behave consistently with parsing DateTime from String
/// E.g. "select cast('1984-01-01 99:99:99' as DateTime);" returns "1984-01-05 04:40:39"
if (unlikely(std::isnan(year) || std::isnan(month) || std::isnan(day_of_month) ||
std::isnan(hour) || std::isnan(minute) || std::isnan(second) ||
year < DATE_LUT_MIN_YEAR || month < 1 || month > 12 || day_of_month < 1 || day_of_month > 31 ||
hour < 0 || hour > 99 || minute < 0 || minute > 99 || second < 0 || second > 99))
return minDateTime(lut);
if (unlikely(year > DATE_LUT_MAX_YEAR))
return maxDateTime(lut);
return lut.makeDateTime(year, month, day_of_month, hour, minute, second);
}
static Int64 minDateTime(const DateLUTImpl & lut)
{
return lut.makeDateTime(DATE_LUT_MIN_YEAR - 1, 1, 1, 0, 0, 0);
}
static Int64 maxDateTime(const DateLUTImpl & lut)
{
return lut.makeDateTime(DATE_LUT_MAX_YEAR + 1, 1, 1, 23, 59, 59);
}
std::string extractTimezone(const ColumnWithTypeAndName & timezone_argument) const
{
std::string timezone;
if (!isStringOrFixedString(timezone_argument.type) || !timezone_argument.column || (timezone_argument.column->size() != 1 && !typeid_cast<const ColumnConst*>(timezone_argument.column.get())))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Argument 'timezone' for function {} must be const string", getName());
timezone = timezone_argument.column->getDataAt(0).toString();
return timezone;
}
};
/// makeDateTime(year, month, day, hour, minute, second, [timezone])
class FunctionMakeDateTime : public FunctionMakeDateTimeBase
{
private:
static constexpr std::array<const char*, 1> optional_argument_names = {"timezone"};
public:
static constexpr auto name = "makeDateTime";
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionMakeDateTime>(); }
String getName() const override { return name; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
checkRequiredArguments(arguments, optional_argument_names.size());
/// Optional timezone argument
std::string timezone;
if (arguments.size() == argument_names.size() + 1)
timezone = extractTimezone(arguments.back());
return std::make_shared<DataTypeDateTime>(timezone);
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
/// Optional timezone argument
std::string timezone;
if (arguments.size() == argument_names.size() + 1)
timezone = extractTimezone(arguments.back());
Columns converted_arguments;
convertRequiredArguments(arguments, converted_arguments);
auto res_column = ColumnUInt32::create(input_rows_count);
auto & result_data = res_column->getData();
const auto & year_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[0]).getData();
const auto & month_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[1]).getData();
const auto & day_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[2]).getData();
const auto & hour_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[3]).getData();
const auto & minute_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[4]).getData();
const auto & second_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[5]).getData();
const auto & date_lut = DateLUT::instance(timezone);
for (size_t i = 0; i < input_rows_count; ++i)
{
const auto year = year_data[i];
const auto month = month_data[i];
const auto day = day_data[i];
const auto hour = hour_data[i];
const auto minute = minute_data[i];
const auto second = second_data[i];
auto date_time = dateTime(year, month, day, hour, minute, second, date_lut);
if (unlikely(date_time < 0))
date_time = 0;
else if (unlikely(date_time > 0x0ffffffffll))
date_time = 0x0ffffffffll;
result_data[i] = date_time;
}
return res_column;
}
};
/// makeDateTime64(year, month, day, hour, minute, second, [fraction], [precision], [timezone])
class FunctionMakeDateTime64 : public FunctionMakeDateTimeBase
{
private:
static constexpr std::array<const char*, 3> optional_argument_names = {"fraction", "precision", "timezone"};
static constexpr UInt8 DEFAULT_PRECISION = 3;
public:
static constexpr auto name = "makeDateTime64";
static FunctionPtr create(ContextPtr) { return std::make_shared<FunctionMakeDateTime64>(); }
String getName() const override { return name; }
DataTypePtr getReturnTypeImpl(const ColumnsWithTypeAndName & arguments) const override
{
checkRequiredArguments(arguments, optional_argument_names.size());
if (arguments.size() >= argument_names.size() + 1)
{
const auto& fraction_argument = arguments[argument_names.size()];
if (!isNumber(fraction_argument.type))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Argument 'fraction' for function {} must be number", getName());
}
/// Optional precision argument
Int64 precision = DEFAULT_PRECISION;
if (arguments.size() >= argument_names.size() + 2)
precision = extractPrecision(arguments[argument_names.size() + 1]);
/// Optional timezone argument
std::string timezone;
if (arguments.size() == argument_names.size() + 3)
timezone = extractTimezone(arguments.back());
return std::make_shared<DataTypeDateTime64>(precision, timezone);
}
ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr &, size_t input_rows_count) const override
{
/// Optional precision argument
Int64 precision = DEFAULT_PRECISION;
if (arguments.size() >= argument_names.size() + 2)
precision = extractPrecision(arguments[argument_names.size() + 1]);
/// Optional timezone argument
std::string timezone;
if (arguments.size() == argument_names.size() + 3)
timezone = extractTimezone(arguments.back());
Columns converted_arguments;
convertRequiredArguments(arguments, converted_arguments);
/// Optional fraction argument
const ColumnVector<Float64>::Container * fraction_data = nullptr;
if (arguments.size() >= argument_names.size() + 1)
{
ColumnPtr fraction_column = castColumn(arguments[argument_names.size()], std::make_shared<DataTypeFloat64>());
fraction_column = fraction_column->convertToFullColumnIfConst();
converted_arguments.push_back(fraction_column);
fraction_data = &typeid_cast<const ColumnFloat64 &>(*converted_arguments[6]).getData();
}
auto res_column = ColumnDecimal<DateTime64>::create(input_rows_count, precision);
auto & result_data = res_column->getData();
const auto & year_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[0]).getData();
const auto & month_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[1]).getData();
const auto & day_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[2]).getData();
const auto & hour_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[3]).getData();
const auto & minute_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[4]).getData();
const auto & second_data = typeid_cast<const ColumnFloat32 &>(*converted_arguments[5]).getData();
const auto & date_lut = DateLUT::instance(timezone);
const auto max_fraction = pow(10, precision) - 1;
const auto min_date_time = minDateTime(date_lut);
const auto max_date_time = maxDateTime(date_lut);
for (size_t i = 0; i < input_rows_count; ++i)
{
const auto year = year_data[i];
const auto month = month_data[i];
const auto day = day_data[i];
const auto hour = hour_data[i];
const auto minute = minute_data[i];
const auto second = second_data[i];
auto date_time = dateTime(year, month, day, hour, minute, second, date_lut);
double fraction = 0;
if (unlikely(date_time == min_date_time))
fraction = 0;
else if (unlikely(date_time == max_date_time))
fraction = 999999999ll;
else
{
fraction = fraction_data ? (*fraction_data)[i] : 0;
if (unlikely(std::isnan(fraction)))
{
date_time = min_date_time;
fraction = 0;
}
else if (unlikely(fraction < 0))
fraction = 0;
else if (unlikely(fraction > max_fraction))
fraction = max_fraction;
}
result_data[i] = DecimalUtils::decimalFromComponents<DateTime64>(date_time, fraction, precision);
}
return res_column;
}
private:
UInt8 extractPrecision(const ColumnWithTypeAndName & precision_argument) const
{
Int64 precision = DEFAULT_PRECISION;
if (!isNumber(precision_argument.type) || !precision_argument.column || (precision_argument.column->size() != 1 && !typeid_cast<const ColumnConst*>(precision_argument.column.get())))
throw Exception(ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Argument 'precision' for function {} must be constant number", getName());
precision = precision_argument.column->getInt(0);
if (precision < 0 || precision > 9)
throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND,
"Argument 'precision' for function {} must be in range [0, 9]", getName());
return precision;
}
};
}
void registerFunctionsMakeDate(FunctionFactory & factory)
{
factory.registerFunction<FunctionMakeDate<MakeDateTraits>>();
factory.registerFunction<FunctionMakeDate<MakeDate32Traits>>();
factory.registerFunction<FunctionMakeDateTime>();
factory.registerFunction<FunctionMakeDateTime64>();
}
}

22
src/IO/ParallelReadBuffer.cpp
View File

@ -33,6 +33,7 @@ bool ParallelReadBuffer::addReaderToPool(std::unique_lock<std::mutex> & /*buffer
auto worker = read_workers.emplace_back(std::make_shared<ReadWorker>(std::move(reader)));
++active_working_reader;
schedule([this, worker = std::move(worker)]() mutable { readerThreadFunction(std::move(worker)); });
return true;
@ -203,18 +204,9 @@ bool ParallelReadBuffer::nextImpl()
void ParallelReadBuffer::readerThreadFunction(ReadWorkerPtr read_worker)
{
{
std::lock_guard lock{mutex};
++active_working_reader;
}
SCOPE_EXIT({
std::lock_guard lock{mutex};
--active_working_reader;
if (active_working_reader == 0)
{
readers_done.notify_all();
}
if (active_working_reader.fetch_sub(1) == 1)
active_working_reader.notify_all();
});
try
@ -269,8 +261,12 @@ void ParallelReadBuffer::finishAndWait()
{
emergency_stop = true;
std::unique_lock lock{mutex};
readers_done.wait(lock, [&] { return active_working_reader == 0; });
size_t active_readers = active_working_reader.load();
while (active_readers != 0)
{
active_working_reader.wait(active_readers);
active_readers = active_working_reader.load();
}
}
}

4
src/IO/ParallelReadBuffer.h
View File

@ -135,9 +135,7 @@ private:
Segment current_segment;
size_t max_working_readers;
size_t active_working_reader{0};
// Triggered when all reader workers are done
std::condition_variable readers_done;
std::atomic_size_t active_working_reader{0};
CallbackRunner schedule;

2
src/IO/ReadBufferFromFileDescriptor.cpp
View File

@ -191,7 +191,7 @@ off_t ReadBufferFromFileDescriptor::seek(off_t offset, int whence)
off_t res = ::lseek(fd, seek_pos, SEEK_SET);
if (-1 == res)
throwFromErrnoWithPath("Cannot seek through file " + getFileName(), getFileName(),
throwFromErrnoWithPath(fmt::format("Cannot seek through file {} at offset {}", getFileName(), seek_pos), getFileName(),
ErrorCodes::CANNOT_SEEK_THROUGH_FILE);
/// Also note that seeking past the file size is not allowed.

2
src/Interpreters/DatabaseCatalog.cpp
View File

@ -12,7 +12,7 @@
#include <Parsers/formatAST.h>
#include <IO/ReadHelpers.h>
#include <Poco/DirectoryIterator.h>
#include <Common/renameat2.h>
#include <Common/atomicRename.h>
#include <Common/CurrentMetrics.h>
#include <base/logger_useful.h>
#include <Poco/Util/AbstractConfiguration.h>

33
src/Interpreters/ExpressionAnalyzer.cpp
View File

@ -9,6 +9,7 @@
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTWindowDefinition.h>
#include <Parsers/DumpASTNode.h>
#include <Parsers/ASTInterpolateElement.h>
#include <DataTypes/DataTypeNullable.h>
#include <Columns/IColumn.h>
@ -1333,6 +1334,38 @@ ActionsDAGPtr SelectQueryExpressionAnalyzer::appendOrderBy(ExpressionActionsChai
with_fill = true;
}
if (auto interpolate_list = select_query->interpolate())
{
NameSet select;
for (const auto & child : select_query->select()->children)
select.insert(child->getAliasOrColumnName());
/// collect columns required for interpolate expressions -
/// interpolate expression can use any available column
auto find_columns = [&step, &select](IAST * function)
{
auto f_impl = [&step, &select](IAST * fn, auto fi)
{
if (auto * ident = fn->as<ASTIdentifier>())
{
/// exclude columns from select expression - they are already available
if (select.count(ident->getColumnName()) == 0)
step.addRequiredOutput(ident->getColumnName());
return;
}
if (fn->as<ASTFunction>() || fn->as<ASTExpressionList>())
for (const auto & ch : fn->children)
fi(ch.get(), fi);
return;
};
f_impl(function, f_impl);
};
for (const auto & interpolate : interpolate_list->children)
find_columns(interpolate->as<ASTInterpolateElement>()->expr.get());
}
if (optimize_read_in_order)
{
for (auto & child : select_query->orderBy()->children)

50
src/Interpreters/FillingRow.cpp
View File

@ -19,26 +19,27 @@ bool equals(const Field & lhs, const Field & rhs)
}
FillingRow::FillingRow(const SortDescription & sort_description) : description(sort_description)
FillingRow::FillingRow(const SortDescription & sort_description_)
: sort_description(sort_description_)
{
row.resize(description.size());
row.resize(sort_description.size());
}
bool FillingRow::operator<(const FillingRow & other) const
{
for (size_t i = 0; i < size(); ++i)
for (size_t i = 0; i < sort_description.size(); ++i)
{
if (row[i].isNull() || other[i].isNull() || equals(row[i], other[i]))
if ((*this)[i].isNull() || other.row[i].isNull() || equals(row[i], other.row[i]))
continue;
return less(row[i], other[i], getDirection(i));
return less(row[i], other.row[i], getDirection(i));
}
return false;
}
bool FillingRow::operator==(const FillingRow & other) const
{
for (size_t i = 0; i < size(); ++i)
if (!equals(row[i], other[i]))
for (size_t i = 0; i < sort_description.size(); ++i)
if (!equals(row[i], other.row[i]))
return false;
return true;
}
@ -48,16 +49,16 @@ bool FillingRow::next(const FillingRow & to_row)
size_t pos = 0;
/// Find position we need to increment for generating next row.
for (; pos < row.size(); ++pos)
if (!row[pos].isNull() && !to_row[pos].isNull() && !equals(row[pos], to_row[pos]))
for (; pos < size(); ++pos)
if (!row[pos].isNull() && !to_row.row[pos].isNull() && !equals(row[pos], to_row.row[pos]))
break;
if (pos == row.size() || less(to_row[pos], row[pos], getDirection(pos)))
if (pos == size() || less(to_row.row[pos], row[pos], getDirection(pos)))
return false;
/// If we have any 'fill_to' value at position greater than 'pos',
/// we need to generate rows up to 'fill_to' value.
for (size_t i = row.size() - 1; i > pos; --i)
for (size_t i = size() - 1; i > pos; --i)
{
if (getFillDescription(i).fill_to.isNull() || row[i].isNull())
continue;
@ -75,21 +76,22 @@ bool FillingRow::next(const FillingRow & to_row)
auto next_value = row[pos];
getFillDescription(pos).step_func(next_value);
if (less(to_row[pos], next_value, getDirection(pos)))
if (less(to_row.row[pos], next_value, getDirection(pos)))
return false;
row[pos] = next_value;
if (equals(row[pos], to_row[pos]))
if (equals(row[pos], to_row.row[pos]))
{
bool is_less = false;
for (size_t i = pos + 1; i < size(); ++i)
size_t i = pos + 1;
for (; i < size(); ++i)
{
const auto & fill_from = getFillDescription(i).fill_from;
if (!fill_from.isNull())
row[i] = fill_from;
else
row[i] = to_row[i];
is_less |= less(row[i], to_row[i], getDirection(i));
row[i] = to_row.row[i];
is_less |= less(row[i], to_row.row[i], getDirection(i));
}
return is_less;
@ -101,12 +103,12 @@ bool FillingRow::next(const FillingRow & to_row)
void FillingRow::initFromDefaults(size_t from_pos)
{
for (size_t i = from_pos; i < row.size(); ++i)
for (size_t i = from_pos; i < sort_description.size(); ++i)
row[i] = getFillDescription(i).fill_from;
}
void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & other_columns, const FillingRow & filling_row)
void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns,
const FillingRow & filling_row, const Block & interpolate_block)
{
for (size_t i = 0; i < filling_columns.size(); ++i)
{
@ -116,6 +118,16 @@ void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & oth
filling_columns[i]->insert(filling_row[i]);
}
if (size_t size = interpolate_block.columns())
{
Columns columns = interpolate_block.getColumns();
for (size_t i = 0; i < size; ++i)
interpolate_columns[i]->insertFrom(*columns[i]->convertToFullColumnIfConst(), 0);
}
else
for (const auto & interpolate_column : interpolate_columns)
interpolate_column->insertDefault();
for (const auto & other_column : other_columns)
other_column->insertDefault();
}

12
src/Interpreters/FillingRow.h
View File

@ -1,5 +1,6 @@
#pragma once
#include <Core/SortDescription.h>
#include <Core/InterpolateDescription.h>
#include <Columns/IColumn.h>
@ -17,7 +18,7 @@ bool equals(const Field & lhs, const Field & rhs);
class FillingRow
{
public:
FillingRow(const SortDescription & sort_description);
explicit FillingRow(const SortDescription & sort_description);
/// Generates next row according to fill 'from', 'to' and 'step' values.
bool next(const FillingRow & to_row);
@ -30,15 +31,16 @@ public:
bool operator<(const FillingRow & other) const;
bool operator==(const FillingRow & other) const;
int getDirection(size_t index) const { return description[index].direction; }
FillColumnDescription & getFillDescription(size_t index) { return description[index].fill_description; }
int getDirection(size_t index) const { return sort_description[index].direction; }
FillColumnDescription & getFillDescription(size_t index) { return sort_description[index].fill_description; }
private:
Row row;
SortDescription description;
SortDescription sort_description;
};
void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & other_columns, const FillingRow & filling_row);
void insertFromFillingRow(MutableColumns & filling_columns, MutableColumns & interpolate_columns, MutableColumns & other_columns,
const FillingRow & filling_row, const Block & interpolate_block);
void copyRowFromColumns(MutableColumns & dest, const Columns & source, size_t row_num);
}

2
src/Interpreters/InterpreterBackupQuery.cpp
View File

@ -41,8 +41,8 @@ namespace
void executeBackup(const ContextPtr & context, const ASTBackupQuery & query)
{
auto backup_settings = BackupSettings::fromBackupQuery(query);
BackupMutablePtr backup = createBackup(BackupInfo::fromAST(*query.backup_name), backup_settings, context);
auto backup_entries = makeBackupEntries(context, query.elements, backup_settings);
BackupMutablePtr backup = createBackup(BackupInfo::fromAST(*query.backup_name), backup_settings, context);
writeBackupEntries(backup, std::move(backup_entries), context->getSettingsRef().max_backup_threads);
}

50
src/Interpreters/InterpreterCreateQuery.cpp
View File

@ -8,7 +8,7 @@
#include <Common/typeid_cast.h>
#include <Common/Macros.h>
#include <Common/randomSeed.h>
#include <Common/renameat2.h>
#include <Common/atomicRename.h>
#include <Common/hex.h>
#include <Core/Defines.h>
@ -54,6 +54,7 @@
#include <DataTypes/DataTypeNullable.h>
#include <DataTypes/DataTypeAggregateFunction.h>
#include <DataTypes/ObjectUtils.h>
#include <DataTypes/hasNullable.h>
#include <Databases/DatabaseFactory.h>
#include <Databases/DatabaseReplicated.h>
@ -480,6 +481,21 @@ ColumnsDescription InterpreterCreateQuery::getColumnsDescription(
{
column_type = makeNullable(column_type);
}
else if (!hasNullable(column_type) &&
col_decl.default_specifier == "DEFAULT" &&
col_decl.default_expression &&
col_decl.default_expression->as<ASTLiteral>() &&
col_decl.default_expression->as<ASTLiteral>()->value.isNull())
{
if (column_type->lowCardinality())
{
const auto * low_cardinality_type = typeid_cast<const DataTypeLowCardinality *>(column_type.get());
assert(low_cardinality_type);
column_type = std::make_shared<DataTypeLowCardinality>(makeNullable(low_cardinality_type->getDictionaryType()));
}
else
column_type = makeNullable(column_type);
}
column_names_and_types.emplace_back(col_decl.name, column_type);
}
@ -1055,6 +1071,38 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create)
/// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way.
TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create);
/// Check type compatible for materialized dest table and select columns
if (create.select && create.is_materialized_view && create.to_table_id)
{
if (StoragePtr to_table = DatabaseCatalog::instance().tryGetTable(
{create.to_table_id.database_name, create.to_table_id.table_name, create.to_table_id.uuid},
getContext()
))
{
Block input_block = InterpreterSelectWithUnionQuery(
create.select->clone(), getContext(), SelectQueryOptions().analyze()).getSampleBlock();
Block output_block = to_table->getInMemoryMetadataPtr()->getSampleBlock();
ColumnsWithTypeAndName input_columns;
ColumnsWithTypeAndName output_columns;
for (const auto & input_column : input_block)
{
if (const auto * output_column = output_block.findByName(input_column.name))
{
input_columns.push_back(input_column.cloneEmpty());
output_columns.push_back(output_column->cloneEmpty());
}
}
ActionsDAG::makeConvertingActions(
input_columns,
output_columns,
ActionsDAG::MatchColumnsMode::Position
);
}
}
DatabasePtr database;
bool need_add_to_database = !create.temporary;
if (need_add_to_database)

79
src/Interpreters/InterpreterSelectQuery.cpp
View File

@ -5,6 +5,7 @@
#include <Parsers/ASTIdentifier.h>
#include <Parsers/ASTLiteral.h>
#include <Parsers/ASTOrderByElement.h>
#include <Parsers/ASTInterpolateElement.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTSelectIntersectExceptQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
@ -100,6 +101,7 @@ namespace ErrorCodes
extern const int INVALID_LIMIT_EXPRESSION;
extern const int INVALID_WITH_FILL_EXPRESSION;
extern const int ACCESS_DENIED;
extern const int UNKNOWN_IDENTIFIER;
}
/// Assumes `storage` is set and the table filter (row-level security) is not empty.
@ -780,6 +782,7 @@ static std::pair<Field, std::optional<IntervalKind>> getWithFillStep(const ASTPt
static FillColumnDescription getWithFillDescription(const ASTOrderByElement & order_by_elem, ContextPtr context)
{
FillColumnDescription descr;
if (order_by_elem.fill_from)
descr.fill_from = getWithFillFieldValue(order_by_elem.fill_from, context);
if (order_by_elem.fill_to)
@ -835,7 +838,6 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, ContextP
std::shared_ptr<Collator> collator;
if (order_by_elem.collation)
collator = std::make_shared<Collator>(order_by_elem.collation->as<ASTLiteral &>().value.get<String>());
if (order_by_elem.with_fill)
{
FillColumnDescription fill_desc = getWithFillDescription(order_by_elem, context);
@ -848,6 +850,77 @@ static SortDescription getSortDescription(const ASTSelectQuery & query, ContextP
return order_descr;
}
static InterpolateDescriptionPtr getInterpolateDescription(
const ASTSelectQuery & query, const Block & source_block, const Block & result_block, const Aliases & aliases, ContextPtr context)
{
InterpolateDescriptionPtr interpolate_descr;
if (query.interpolate())
{
NamesAndTypesList source_columns;
ColumnsWithTypeAndName result_columns;
ASTPtr exprs = std::make_shared<ASTExpressionList>();
if (query.interpolate()->children.empty())
{
std::unordered_map<String, DataTypePtr> column_names;
for (const auto & column : result_block.getColumnsWithTypeAndName())
column_names[column.name] = column.type;
for (const auto & elem : query.orderBy()->children)
if (elem->as<ASTOrderByElement>()->with_fill)
column_names.erase(elem->as<ASTOrderByElement>()->children.front()->getColumnName());
for (const auto & [name, type] : column_names)
{
source_columns.emplace_back(name, type);
result_columns.emplace_back(type, name);
exprs->children.emplace_back(std::make_shared<ASTIdentifier>(name));
}
}
else
{
NameSet col_set;
for (const auto & elem : query.interpolate()->children)
{
const auto & interpolate = elem->as<ASTInterpolateElement &>();
if (const ColumnWithTypeAndName *result_block_column = result_block.findByName(interpolate.column))
{
if (!col_set.insert(result_block_column->name).second)
throw Exception(ErrorCodes::INVALID_WITH_FILL_EXPRESSION,
"Duplicate INTERPOLATE column '{}'", interpolate.column);
result_columns.emplace_back(result_block_column->type, result_block_column->name);
}
else
throw Exception(ErrorCodes::UNKNOWN_IDENTIFIER,
"Missing column '{}' as an INTERPOLATE expression target", interpolate.column);
exprs->children.emplace_back(interpolate.expr->clone());
}
col_set.clear();
for (const auto & column : source_block)
{
source_columns.emplace_back(column.name, column.type);
col_set.insert(column.name);
}
for (const auto & column : result_block)
if (col_set.count(column.name) == 0)
source_columns.emplace_back(column.name, column.type);
}
auto syntax_result = TreeRewriter(context).analyze(exprs, source_columns);
ExpressionAnalyzer analyzer(exprs, syntax_result, context);
ActionsDAGPtr actions = analyzer.getActionsDAG(true);
ActionsDAGPtr conv_dag = ActionsDAG::makeConvertingActions(actions->getResultColumns(),
result_columns, ActionsDAG::MatchColumnsMode::Position, true);
ActionsDAGPtr merge_dag = ActionsDAG::merge(std::move(*actions->clone()), std::move(*conv_dag));
interpolate_descr = std::make_shared<InterpolateDescription>(merge_dag, aliases);
}
return interpolate_descr;
}
static SortDescription getSortDescriptionFromGroupBy(const ASTSelectQuery & query)
{
SortDescription order_descr;
@ -2515,7 +2588,9 @@ void InterpreterSelectQuery::executeWithFill(QueryPlan & query_plan)
if (fill_descr.empty())
return;
auto filling_step = std::make_unique<FillingStep>(query_plan.getCurrentDataStream(), std::move(fill_descr));
InterpolateDescriptionPtr interpolate_descr =
getInterpolateDescription(query, source_header, result_header, syntax_analyzer_result->aliases, context);
auto filling_step = std::make_unique<FillingStep>(query_plan.getCurrentDataStream(), std::move(fill_descr), interpolate_descr);
query_plan.addStep(std::move(filling_step));
}
}

27
src/Interpreters/MergeTreeTransaction.cpp
View File

@ -250,21 +250,6 @@ bool MergeTreeTransaction::rollback() noexcept
/// Discard changes in active parts set
/// Remove parts that were created, restore parts that were removed (except parts that were created by this transaction too)
for (const auto & part : parts_to_remove)
{
if (part->version.isRemovalTIDLocked())
{
/// Don't need to remove part from working set if it was created and removed by this transaction
assert(part->version.removal_tid_lock == tid.getHash());
continue;
}
/// FIXME do not lock removal_tid when rolling back part creation, it's ugly
const_cast<MergeTreeData &>(part->storage).removePartsFromWorkingSet(NO_TRANSACTION_RAW, {part}, true);
}
for (const auto & part : parts_to_activate)
if (part->version.getCreationTID() != tid)
const_cast<MergeTreeData &>(part->storage).restoreAndActivatePart(part);
/// Kind of optimization: cleanup thread can remove these parts immediately
for (const auto & part : parts_to_remove)
@ -274,6 +259,18 @@ bool MergeTreeTransaction::rollback() noexcept
part->appendCSNToVersionMetadata(VersionMetadata::CREATION);
}
for (const auto & part : parts_to_remove)
{
/// NOTE It's possible that part is already removed from working set in the same transaction
/// (or, even worse, in a separate non-transactional query with PrehistoricTID),
/// but it's not a problem: removePartsFromWorkingSet(...) will do nothing in this case.
const_cast<MergeTreeData &>(part->storage).removePartsFromWorkingSet(NO_TRANSACTION_RAW, {part}, true);
}
for (const auto & part : parts_to_activate)
if (part->version.getCreationTID() != tid)
const_cast<MergeTreeData &>(part->storage).restoreAndActivatePart(part);
for (const auto & part : parts_to_activate)
{
/// Clear removal_tid from version metadata file, so we will not need to distinguish TIDs that were not committed

2
src/Interpreters/ProfileEventsExt.cpp
View File

@ -105,7 +105,7 @@ void getProfileEvents(
{"value", std::make_shared<DataTypeInt64>()},
};
ColumnsWithTypeAndName temp_columns;
ColumnsWithTypeAndName temp_columns;
for (auto const & name_and_type : column_names_and_types)
temp_columns.emplace_back(name_and_type.type, name_and_type.name);

4
src/Interpreters/QueryNormalizer.cpp
View File

@ -9,6 +9,7 @@
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTQueryParameter.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTInterpolateElement.h>
#include <Common/StringUtils/StringUtils.h>
#include <Common/quoteString.h>
#include <IO/WriteHelpers.h>
@ -134,7 +135,8 @@ void QueryNormalizer::visit(ASTTablesInSelectQueryElement & node, const ASTPtr &
static bool needVisitChild(const ASTPtr & child)
{
return !(child->as<ASTSelectQuery>() || child->as<ASTTableExpression>());
/// exclude interpolate elements - they are not subject for normalization and will be processed in filling transform
return !(child->as<ASTSelectQuery>() || child->as<ASTTableExpression>() || child->as<ASTInterpolateElement>());
}
/// special visitChildren() for ASTSelectQuery

30
src/Interpreters/RequiredSourceColumnsVisitor.cpp
View File

@ -7,6 +7,7 @@
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSubquery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTInterpolateElement.h>
namespace DB
{
@ -46,7 +47,7 @@ bool RequiredSourceColumnsMatcher::needChildVisit(const ASTPtr & node, const AST
return false;
/// Processed. Do not need children.
if (node->as<ASTTableExpression>() || node->as<ASTArrayJoin>() || node->as<ASTSelectQuery>())
if (node->as<ASTTableExpression>() || node->as<ASTArrayJoin>() || node->as<ASTSelectQuery>() || node->as<ASTInterpolateElement>())
return false;
if (const auto * f = node->as<ASTFunction>())
@ -114,15 +115,42 @@ void RequiredSourceColumnsMatcher::visit(const ASTPtr & ast, Data & data)
void RequiredSourceColumnsMatcher::visit(const ASTSelectQuery & select, const ASTPtr &, Data & data)
{
NameSet select_columns;
/// special case for top-level SELECT items: they are publics
for (auto & node : select.select()->children)
{
select_columns.insert(node->getAliasOrColumnName());
if (const auto * identifier = node->as<ASTIdentifier>())
data.addColumnIdentifier(*identifier);
else
data.addColumnAliasIfAny(*node);
}
if (auto interpolate_list = select.interpolate())
{
auto find_columns = [&data, &select_columns](IAST * function)
{
auto f_impl = [&data, &select_columns](IAST * fn, auto fi)
{
if (auto * ident = fn->as<ASTIdentifier>())
{
if (select_columns.count(ident->getColumnName()) == 0)
data.addColumnIdentifier(*ident);
return;
}
if (fn->as<ASTFunction>() || fn->as<ASTExpressionList>())
for (const auto & ch : fn->children)
fi(ch.get(), fi);
return;
};
f_impl(function, f_impl);
};
for (const auto & interpolate : interpolate_list->children)
find_columns(interpolate->as<ASTInterpolateElement>()->expr.get());
}
if (const auto & with = select.with())
{
for (auto & node : with->children)

9
src/Interpreters/TransactionLog.cpp
View File

@ -212,7 +212,7 @@ void TransactionLog::runUpdatingThread()
if (stop_flag.load())
return;
if (!zookeeper)
if (getZooKeeper()->expired())
{
auto new_zookeeper = global_context->getZooKeeper();
std::lock_guard lock{mutex};
@ -222,16 +222,11 @@ void TransactionLog::runUpdatingThread()
loadNewEntries();
removeOldEntries();
}
catch (const Coordination::Exception & e)
catch (const Coordination::Exception &)
{
tryLogCurrentException(log);
/// TODO better backoff
std::this_thread::sleep_for(std::chrono::milliseconds(1000));
if (Coordination::isHardwareError(e.code))
{
std::lock_guard lock{mutex};
zookeeper.reset();
}
log_updated_event->set();
}
catch (...)

26
src/Interpreters/TreeRewriter.cpp
View File

@ -32,6 +32,7 @@
#include <Parsers/ASTSelectQuery.h>
#include <Parsers/ASTSelectWithUnionQuery.h>
#include <Parsers/ASTTablesInSelectQuery.h>
#include <Parsers/ASTInterpolateElement.h>
#include <Parsers/queryToString.h>
#include <DataTypes/NestedUtils.h>
@ -420,7 +421,8 @@ void renameDuplicatedColumns(const ASTSelectQuery * select_query)
/// Sometimes we have to calculate more columns in SELECT clause than will be returned from query.
/// This is the case when we have DISTINCT or arrayJoin: we require more columns in SELECT even if we need less columns in result.
/// Also we have to remove duplicates in case of GLOBAL subqueries. Their results are placed into tables so duplicates are impossible.
void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query, const Names & required_result_columns, bool remove_dups)
/// Also remove all INTERPOLATE columns which are not in SELECT anymore.
void removeUnneededColumnsFromSelectClause(ASTSelectQuery * select_query, const Names & required_result_columns, bool remove_dups)
{
ASTs & elements = select_query->select()->children;
@ -449,6 +451,8 @@ void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query,
ASTs new_elements;
new_elements.reserve(elements.size());
NameSet remove_columns;
for (const auto & elem : elements)
{
String name = elem->getAliasOrColumnName();
@ -465,6 +469,8 @@ void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query,
}
else
{
remove_columns.insert(name);
ASTFunction * func = elem->as<ASTFunction>();
/// Never remove untuple. It's result column may be in required columns.
@ -478,6 +484,24 @@ void removeUnneededColumnsFromSelectClause(const ASTSelectQuery * select_query,
}
}
if (select_query->interpolate())
{
auto & children = select_query->interpolate()->children;
if (!children.empty())
{
for (auto it = children.begin(); it != children.end();)
{
if (remove_columns.count((*it)->as<ASTInterpolateElement>()->column))
it = select_query->interpolate()->children.erase(it);
else
++it;
}
if (children.empty())
select_query->setExpression(ASTSelectQuery::Expression::INTERPOLATE, nullptr);
}
}
elements = std::move(new_elements);
}

10
src/Interpreters/UserDefinedExecutableFunctionFactory.cpp
View File

@ -4,8 +4,6 @@
#include <Common/filesystemHelpers.h>
#include <IO/WriteHelpers.h>
#include <Processors/Sources/ShellCommandSource.h>
#include <Processors/Sources/SourceFromSingleChunk.h>
#include <Formats/formatBlock.h>
@ -78,12 +76,18 @@ public:
command,
user_scripts_path);
if (!std::filesystem::exists(std::filesystem::path(script_path)))
if (!FS::exists(script_path))
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Executable file {} does not exist inside user scripts folder {}",
command,
user_scripts_path);
if (!FS::canExecute(script_path))
throw Exception(ErrorCodes::UNSUPPORTED_METHOD,
"Executable file {} is not executable inside user scripts folder {}",
command,
user_scripts_path);
command = std::move(script_path);
}

4
src/Interpreters/ZooKeeperLog.cpp
View File

@ -116,6 +116,8 @@ NamesAndTypesList ZooKeeperLogElement::getNamesAndTypes()
{"type", std::move(type_enum)},
{"event_date", std::make_shared<DataTypeDate>()},
{"event_time", std::make_shared<DataTypeDateTime64>(6)},
{"thread_id", std::make_shared<DataTypeUInt64>()},
{"query_id", std::make_shared<DataTypeString>()},
{"address", DataTypeFactory::instance().get("IPv6")},
{"port", std::make_shared<DataTypeUInt16>()},
{"session_id", std::make_shared<DataTypeInt64>()},
@ -164,6 +166,8 @@ void ZooKeeperLogElement::appendToBlock(MutableColumns & columns) const
auto event_time_seconds = event_time / 1000000;
columns[i++]->insert(DateLUT::instance().toDayNum(event_time_seconds).toUnderType());
columns[i++]->insert(event_time);
columns[i++]->insert(thread_id);
columns[i++]->insert(query_id);
columns[i++]->insertData(IPv6ToBinary(address.host()).data(), 16);
columns[i++]->insert(address.port());
columns[i++]->insert(session_id);

2
src/Interpreters/ZooKeeperLog.h
View File

@ -22,6 +22,8 @@ struct ZooKeeperLogElement
Type type = UNKNOWN;
Decimal64 event_time = 0;
UInt64 thread_id = 0;
String query_id;
Poco::Net::SocketAddress address;
Int64 session_id = 0;

16
src/Parsers/ASTInterpolateElement.cpp Normal file
View File

@ -0,0 +1,16 @@
#include <Columns/Collator.h>
#include <Parsers/ASTInterpolateElement.h>
#include <Common/SipHash.h>
#include <IO/Operators.h>
namespace DB
{
void ASTInterpolateElement::formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const
{
settings.ostr << column << (settings.hilite ? hilite_keyword : "") << " AS " << (settings.hilite ? hilite_none : "");
expr->formatImpl(settings, state, frame);
}
}

31
src/Parsers/ASTInterpolateElement.h Normal file
View File

@ -0,0 +1,31 @@
#pragma once
#include <Parsers/IAST.h>
namespace DB
{
class ASTInterpolateElement : public IAST
{
public:
String column;
ASTPtr expr;
String getID(char delim) const override { return String("InterpolateElement") + delim + "(column " + column + ")"; }
ASTPtr clone() const override
{
auto clone = std::make_shared<ASTInterpolateElement>(*this);
clone->expr = clone->expr->clone();
clone->children.clear();
clone->children.push_back(clone->expr);
return clone;
}
protected:
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
};
}

1
src/Parsers/ASTOrderByElement.h
View File

@ -37,4 +37,5 @@ public:
protected:
void formatImpl(const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override;
};
}

11
src/Parsers/ASTSelectQuery.cpp
View File

@ -129,6 +129,17 @@ void ASTSelectQuery::formatImpl(const FormatSettings & s, FormatState & state, F
s.one_line
? orderBy()->formatImpl(s, state, frame)
: orderBy()->as<ASTExpressionList &>().formatImplMultiline(s, state, frame);
if (interpolate())
{
s.ostr << (s.hilite ? hilite_keyword : "") << s.nl_or_ws << indent_str << "INTERPOLATE" << (s.hilite ? hilite_none : "");
if (!interpolate()->children.empty())
{
s.ostr << " (";
interpolate()->formatImpl(s, state, frame);
s.ostr << " )";
}
}
}
if (limitByLength())

Some files were not shown because too many files have changed in this diff Show More