Merge pull request #51603 from Avogar/full-filename-placeholder

Add placeholder for full filename in rename_files_after_processing setting

docs/en/operations/settings/settings.md

@@ -4524,6 +4524,7 @@ This setting allows to specify renaming pattern for files processed by `file` ta
 
 ### Placeholders
 
+- `%a` — Full original filename (e.g., "sample.csv").
 - `%f` — Original filename without extension (e.g., "sample").
 - `%e` — Original file extension with dot (e.g., ".csv").
 - `%t` — Timestamp (in microseconds).

docs/ru/operations/settings/settings.md

@@ -4201,6 +4201,7 @@ SELECT *, timezone() FROM test_tz WHERE d = '2000-01-01 00:00:00' SETTINGS sessi
 ### Шаблон
 Шаблон поддерживает следующие виды плейсхолдеров:
 
+- `%a` — Полное исходное имя файла (например "sample.csv").
 - `%f` — Исходное имя файла без расширения (например "sample").
 - `%e` — Оригинальное расширение файла с точкой (например ".csv").
 - `%t` — Текущее время (в микросекундах).

src/Common/FileRenamer.cpp

@@ -47,6 +47,7 @@ String FileRenamer::generateNewFilename(const String & filename) const
     // Define placeholders and their corresponding values
     std::map<String, String> placeholders =
     {
+        {"%a", filename},
         {"%f", file_base},
         {"%e", file_ext},
         {"%t", timestamp},
@@ -69,16 +70,17 @@ bool FileRenamer::isEmpty() const
 bool FileRenamer::validateRenamingRule(const String & rule, bool throw_on_error)
 {
     // Check if the rule contains invalid placeholders
-    re2::RE2 invalid_placeholder_pattern("^([^%]|%[fet%])*$");
+    re2::RE2 invalid_placeholder_pattern("^([^%]|%[afet%])*$");
     if (!re2::RE2::FullMatch(rule, invalid_placeholder_pattern))
     {
         if (throw_on_error)
-            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid renaming rule: Allowed placeholders only %f, %e, %t, and %%");
+            throw Exception(ErrorCodes::BAD_ARGUMENTS, "Invalid renaming rule: Allowed placeholders only %a, %f, %e, %t, and %%");
         return false;
     }
 
     // Replace valid placeholders with empty strings and count remaining percentage signs.
     String replaced_rule = rule;
+    boost::replace_all(replaced_rule, "%a", "");
     boost::replace_all(replaced_rule, "%f", "");
     boost::replace_all(replaced_rule, "%e", "");
     boost::replace_all(replaced_rule, "%t", "");

src/Common/FileRenamer.h

@@ -9,6 +9,7 @@ namespace DB
 /**
   * The FileRenamer class provides functionality for renaming files based on given pattern with placeholders
   * The supported placeholders are:
+  *   %a - Full original file name ("sample.csv")
   *   %f - Original filename without extension ("sample")
   *   %e - Original file extension with dot (".csv")
   *   %t - Timestamp (in microseconds)

src/Core/Settings.h

@@ -736,7 +736,7 @@ class IColumn;
     M(String, workload, "default", "Name of workload to be used to access resources", 0) \
     M(Milliseconds, storage_system_stack_trace_pipe_read_timeout_ms, 100, "Maximum time to read from a pipe for receiving information from the threads when querying the `system.stack_trace` table. This setting is used for testing purposes and not meant to be changed by users.", 0) \
     \
-    M(String, rename_files_after_processing, "", "Rename successfully processed files according to the specified pattern; Pattern can include the following placeholders: `%f` (original filename without extension), `%e` (file extension with dot), `%t` (current timestamp in µs), and `%%` (% sign)", 0) \
+    M(String, rename_files_after_processing, "", "Rename successfully processed files according to the specified pattern; Pattern can include the following placeholders: `%a` (full original file name), `%f` (original filename without extension), `%e` (file extension with dot), `%t` (current timestamp in µs), and `%%` (% sign)", 0) \
     \
     M(Bool, parallelize_output_from_storages, true, "Parallelize output for reading step from storage. It allows parallelizing query processing right after reading from storage if possible", 0) \
     M(String, insert_deduplication_token, "", "If not empty, used for duplicate detection instead of data digest", 0) \

tests/queries/0_stateless/02732_rename_after_processing.reference

@@ -19,3 +19,6 @@ OK
 tmp5.csv
 OK
 tmp5.csv
+4
+tmp6.csv.processed
+!tmp6.csv

tests/queries/0_stateless/02732_rename_after_processing.sh

@@ -29,6 +29,7 @@ cp ${tmp_dir}/tmp.csv ${tmp_dir}/tmp3_1.csv
 cp ${tmp_dir}/tmp.csv ${tmp_dir}/tmp3_2.csv
 cp ${tmp_dir}/tmp.csv ${tmp_dir}/tmp4.csv
 cp ${tmp_dir}/tmp.csv ${tmp_dir}/tmp5.csv
+cp ${tmp_dir}/tmp.csv ${tmp_dir}/tmp6.csv
 
 ### Checking that renaming works
 
@@ -115,5 +116,14 @@ if [ -e "${tmp_dir}/tmp5.csv" ]; then
     echo "tmp5.csv"
 fi
 
+# check full file name placeholder
+${CLICKHOUSE_CLIENT} --rename-files-after-processing="%a.processed" -q "SELECT COUNT(*) FROM file('${unique_name}/tmp6.csv')"
+if [ -e "${tmp_dir}/tmp6.csv.processed" ]; then
+  echo "tmp6.csv.processed"
+fi
+if [ ! -e "${tmp_dir}/tmp6.csv" ]; then
+    echo "!tmp6.csv"
+fi
+
 # Clean
 rm -rd $tmp_dir