diff --git a/scripts/settings/autogenerate-settings.sh b/scripts/settings/autogenerate-settings.sh index 1202812c372..92b0440f4fc 100755 --- a/scripts/settings/autogenerate-settings.sh +++ b/scripts/settings/autogenerate-settings.sh @@ -1,5 +1,4 @@ #!/usr/bin/env bash - # --- Script Configuration --- SCRIPT_NAME=$(basename "$0") SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" @@ -7,7 +6,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" target_dir=$(dirname "$(dirname "$SCRIPT_DIR")") # Should be project root: .../clickhouse-docs # Temporary directory within the target project structure tmp_dir="$target_dir/scripts/tmp" - # --- Parse Command Line Arguments --- CUSTOM_BINARY="" while getopts "b:" opt; do @@ -28,21 +26,17 @@ while getopts "b:" opt; do ;; esac done - # --- Ensure temporary directory exists --- mkdir -p "$tmp_dir" || { echo "[$SCRIPT_NAME] Error: Failed to create temporary directory $tmp_dir"; exit 1; } cd "$tmp_dir" || { echo "[$SCRIPT_NAME] Error: Failed to change directory to $tmp_dir"; exit 1; } echo "[$SCRIPT_NAME] Changed working directory to: $(pwd)" - # --- Target Binary Configuration --- script_filename="clickhouse" script_path="./$script_filename" # Relative path within tmp_dir - # --- Helper Functions --- command_exists() { command -v "$1" &> /dev/null } - # Function to get architecture name for download URLs (handles macOS arm64) get_arch() { local arch_raw @@ -54,17 +48,14 @@ get_arch() { *) echo "" ;; # Return empty for other architectures esac } - # Function to verify a ClickHouse binary verify_clickhouse_binary() { local binary_path="$1" echo "[$SCRIPT_NAME] Verifying ClickHouse binary at '$binary_path'..." - if [[ ! -f "$binary_path" ]]; then echo "[$SCRIPT_NAME] Error: Binary file not found at '$binary_path'" return 1 fi - if [[ ! -x "$binary_path" ]]; then echo "[$SCRIPT_NAME] Setting executable permission on binary..." chmod +x "$binary_path" || { @@ -72,11 +63,9 @@ verify_clickhouse_binary() { return 1; } fi - local version_output version_output=$("$binary_path" --version 2> /dev/null) local exit_code=$? - if [ $exit_code -eq 0 ]; then echo "[$SCRIPT_NAME] ClickHouse binary verified successfully at '$binary_path'" echo "[$SCRIPT_NAME] Version info: $version_output" @@ -87,34 +76,50 @@ verify_clickhouse_binary() { return 1 # Failure fi } - # Function to setup a custom binary setup_custom_binary() { local source_path="$1" - # Verify the provided binary first if ! verify_clickhouse_binary "$source_path"; then echo "[$SCRIPT_NAME] Error: The specified custom binary is not valid." return 1 fi - # Copy the binary to our tmp directory with the expected name echo "[$SCRIPT_NAME] Copying custom binary to '$script_path'..." cp "$source_path" "$script_path" || { echo "[$SCRIPT_NAME] Error: Failed to copy custom binary to tmp directory." return 1 } - chmod +x "$script_path" || { echo "[$SCRIPT_NAME] Error: Failed to set execute permission on copied binary." return 1 } - # Verify the copied binary verify_clickhouse_binary "$script_path" return $? } +# Function to insert content between AUTOGENERATED tags +insert_content_between_tags() { + local src_file="$1" + local dest_file="$2" + + # Create a temporary file + local tmp_file=$(mktemp) + trap 'rm -f "$tmp_file"' EXIT + + # 1. Copy everything up to and including + awk '// {print; exit}; {print}' "$dest_file" > "$tmp_file" + + # 2. Add the content from the source file + cat "$src_file" >> "$tmp_file" + + # 3. Add everything from to the end of the file + awk 'BEGIN {found=0}; // {found=1}; found {print}' "$dest_file" >> "$tmp_file" + + # Replace the original file with the modified content + mv "$tmp_file" "$dest_file" +} # --- Dependency Checks --- for cmd in bash curl uname mv chmod rm mkdir find cat grep sed ls pwd sort head tar; do if ! command_exists $cmd; then @@ -123,33 +128,27 @@ for cmd in bash curl uname mv chmod rm mkdir find cat grep sed ls pwd sort head fi done echo "[$SCRIPT_NAME] All basic dependencies found." - # --- Installation Function 2 (Primary: Official Script) --- install_clickhouse_via_script() { echo "[$SCRIPT_NAME] Attempting install via official script (curl | sh) into current directory ($tmp_dir)..." local install_script_path="./install_clickhouse.sh" local install_success=true - # Download the installer script if ! curl -fsSL -o "$install_script_path" https://clickhouse.com/; then echo "[$SCRIPT_NAME] Error (Script): Failed to download the official install script from https://clickhouse.com/" rm -f "$install_script_path" return 1 fi - echo "[$SCRIPT_NAME] Downloaded install script. Executing..." chmod +x "$install_script_path" || echo "[$SCRIPT_NAME] Warning (Script): Could not chmod install script." - # Execute the downloaded script using bash if ! bash "$install_script_path"; then echo "[$SCRIPT_NAME] Error (Script): The official install script failed. See output above." install_success=false fi rm -f "$install_script_path" # Clean up installer script - if ! $install_success; then return 1; fi echo "[$SCRIPT_NAME] Official install script finished." - # --- Verification Steps --- if [[ ! -f "$script_path" ]]; then # Checks for ./clickhouse echo "[$SCRIPT_NAME] Error (Script): ClickHouse binary not found at '$script_path' after running install script." @@ -157,12 +156,10 @@ install_clickhouse_via_script() { fi echo "[$SCRIPT_NAME] Found binary at '$script_path'." chmod +x "$script_path" || { echo "[$SCRIPT_NAME] Error (Script): Failed to set execute permission on '$script_path'"; return 1; } - echo "[$SCRIPT_NAME] Verifying downloaded binary '$script_path' (Script method)..." local version_output version_output=$("$script_path" --version 2> /dev/null) local exit_code=$? - if [ $exit_code -eq 0 ]; then echo "[$SCRIPT_NAME] ClickHouse binary (Script method) installed and verified successfully at '$script_path'" echo "[$SCRIPT_NAME] Installed version info: $version_output" @@ -173,11 +170,8 @@ install_clickhouse_via_script() { rm -f "$script_path"; return 1; # Failure fi } - # --- Main Script Logic --- - INSTALL_SUCCESS=false - # Check if a custom binary was provided if [[ -n "$CUSTOM_BINARY" ]]; then echo "[$SCRIPT_NAME] Using custom ClickHouse binary: $CUSTOM_BINARY" @@ -215,24 +209,21 @@ else fi fi fi - # Check final install status if ! $INSTALL_SUCCESS; then echo "[$SCRIPT_NAME] FATAL: All installation methods failed." cd .. || echo "[$SCRIPT_NAME] Warning: Failed to cd back to parent directory before exiting." exit 1 fi - # --- Auto-generate Settings Documentation --- echo "[$SCRIPT_NAME] Auto-generating settings markdown pages..." - -# Process other SQL files first (non-function related) -sql_files_found=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.sql' ! -name 'generate-functions.sql' -print -quit) +# Process other SQL files first (non-function and non-system-tables related) +sql_files_found=$(find "$SCRIPT_DIR" -maxdepth 1 -name '*.sql' ! -name 'generate-functions.sql' ! -name 'system-tables.sql' -print -quit) if [ -z "$sql_files_found" ]; then - echo "[$SCRIPT_NAME] Warning: No non-function *.sql files found in $SCRIPT_DIR to process." + echo "[$SCRIPT_NAME] Warning: No settings *.sql files found in $SCRIPT_DIR to process." else for SQL_FILE in "$SCRIPT_DIR"/*.sql; do - if [ -f "$SQL_FILE" ] && [ "$(basename "$SQL_FILE")" != "generate-functions.sql" ]; then + if [ -f "$SQL_FILE" ] && [ "$(basename "$SQL_FILE")" != "generate-functions.sql" ] && [ "$(basename "$SQL_FILE")" != "system-tables.sql" ]; then echo "[$SCRIPT_NAME] Processing SQL file: $SQL_FILE" # Run ClickHouse from CWD ($tmp_dir), passing absolute path to SQL file "$script_path" --queries-file "$SQL_FILE" || { @@ -244,12 +235,10 @@ else fi done fi - # Generate function documentation using the consolidated SQL file FUNCTION_SQL_FILE="$SCRIPT_DIR/generate-functions.sql" if [ -f "$FUNCTION_SQL_FILE" ]; then echo "[$SCRIPT_NAME] Generating function documentation using consolidated SQL file..." - # Define function categories to process FUNCTION_CATEGORIES=( "Arithmetic" @@ -279,7 +268,6 @@ if [ -f "$FUNCTION_SQL_FILE" ]; then "Time Window" "Tuple" ) - for CATEGORY in "${FUNCTION_CATEGORIES[@]}"; do echo "[$SCRIPT_NAME] Processing $CATEGORY functions..." "$script_path" --param_category="$CATEGORY" --queries-file "$FUNCTION_SQL_FILE" || { @@ -288,7 +276,6 @@ if [ -f "$FUNCTION_SQL_FILE" ]; then "$script_path" --param_category="$CATEGORY" --queries-file "$FUNCTION_SQL_FILE" exit 1; } - # Rename the temporary output file to the correct name # Category can be "Dates and Times" for example, resulting in `dates and times-functions.md` so we change spaces to underscores CATEGORY_LOWER=$(echo "$CATEGORY" | tr '[:upper:]' '[:lower:]' | tr ' ' '_') @@ -306,7 +293,6 @@ if [ -f "$FUNCTION_SQL_FILE" ]; then else echo "[$SCRIPT_NAME] Warning: Consolidated function SQL file not found at $FUNCTION_SQL_FILE" fi - # --- temporary sed replacements --- sed -i.bak \ -e 's/Limit the max number of partitions that can be accessed in one query. <= 0 means unlimited./Limit the max number of partitions that can be accessed in one query. `<=` 0 means unlimited./g' \ @@ -315,7 +301,6 @@ sed -i.bak \ -e 's#/#`/`#g' \ -e 's#(/columns and /checksums)#`(/columns and /checksums)`#g' \ "$tmp_dir"/generated_merge_tree_settings.md > /dev/null - # --- Move Generated Files --- # Define destination paths relative to the project root ($target_dir) move_src_files=("settings-formats.md" "settings.md" "server_settings.md") @@ -328,7 +313,6 @@ append_src_files=("generated_merge_tree_settings.md") append_dest_files=( "docs/operations/settings/merge-tree-settings.md" ) - echo "[$SCRIPT_NAME] Moving generated markdown files to documentation directories..." # Iterate using index; declare loop variables WITHOUT 'local' for i in "${!move_src_files[@]}"; do @@ -337,7 +321,6 @@ for i in "${!move_src_files[@]}"; do dest_full_path="$target_dir/${move_dest_files[i]}" # Source file is relative to CWD ($tmp_dir) src_rel_path="./$src_file" - if [ -f "$src_rel_path" ]; then # Ensure destination directory exists mkdir -p "$(dirname "$dest_full_path")" || { echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; exit 1; } @@ -348,7 +331,6 @@ for i in "${!move_src_files[@]}"; do echo "[$SCRIPT_NAME] Warning: Expected output file $src_rel_path not found in $tmp_dir. Skipping move." fi done - echo "[$SCRIPT_NAME] Appending generated markdown files to documentation directories..." # Iterate using index; declare loop variables WITHOUT 'local' for i in "${!append_src_files[@]}"; do @@ -357,7 +339,6 @@ for i in "${!append_src_files[@]}"; do dest_full_path="$target_dir/${append_dest_files[i]}" # Source file is relative to CWD ($tmp_dir) src_rel_path="./$src_file" - if [ -f "$src_rel_path" ]; then # Ensure destination directory exists mkdir -p "$(dirname "$dest_full_path")" || { echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; exit 1; } @@ -370,7 +351,6 @@ for i in "${!append_src_files[@]}"; do echo "[$SCRIPT_NAME] Warning: Expected output file $src_rel_path not found in $tmp_dir. Skipping append." fi done - # --- Append content between tags on the page insert_src_files=( "experimental-beta-settings.md" @@ -401,7 +381,6 @@ insert_src_files=( "time_window-functions.md" "tuple-functions.md" ) - insert_dest_files=( "docs/about-us/beta-and-experimental-features.md" "docs/sql-reference/functions/arithmetic-functions.md" @@ -431,45 +410,24 @@ insert_dest_files=( "docs/sql-reference/functions/time-window-functions.md" "docs/sql-reference/functions/tuple-functions.md" ) - echo "[$SCRIPT_NAME] Inserting generated markdown content between AUTOGENERATED_START and AUTOGENERATED_END tags" for i in "${!insert_src_files[@]}"; do src_file="${insert_src_files[i]}" - # Construct full destination path using $target_dir (project root) dest_full_path="$target_dir/${insert_dest_files[i]}" - # Source file is relative to CWD ($tmp_dir) src_rel_path="./$src_file" - if [ -f "$src_rel_path" ]; then - # Ensure destination directory exists mkdir -p "$(dirname "$dest_full_path")" || { echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; exit 1; } echo "[$SCRIPT_NAME] Appending $src_rel_path to $dest_full_path" - # Replace the content between and tags if grep -q "" "$dest_full_path" && grep -q "" "$dest_full_path"; then - # Create a temporary file - tmp_file=$(mktemp) - trap 'rm -f "$tmp_file"' EXIT - - # 1. Copy everything up to and including - awk '// {print; exit}; {print}' "$dest_full_path" > "$tmp_file" - - # 2. Add the content from the source file - cat "$src_rel_path" >> "$tmp_file" - - # 3. Add everything from to the end of the file - awk 'BEGIN {found=0}; // {found=1}; found {print}' "$dest_full_path" >> "$tmp_file" - - # Replace the original file with the modified content - mv "$tmp_file" "$dest_full_path" + insert_content_between_tags "$src_rel_path" "$dest_full_path" else echo "[$SCRIPT_NAME] Error: Expected to find AUTOGENERATED_START and AUTOGENERATED_END tags in $dest_full_path, but did not" exit 1 fi - # Remove source file after successful append rm -f "$src_rel_path" else @@ -477,6 +435,94 @@ for i in "${!insert_src_files[@]}"; do fi done +# --- Dynamically retrieve system tables list --- +echo "[$SCRIPT_NAME] Retrieving list of system tables from ClickHouse..." + +# Query to get all system tables +SYSTEM_TABLES_QUERY="SELECT name FROM system.tables WHERE database = 'system' ORDER BY name FORMAT TSV" + +# Execute the query and store results in an array (macOS compatible) +SYSTEM_TABLES=() +while IFS= read -r line; do + SYSTEM_TABLES+=("$line") +done < <("$script_path" --query "$SYSTEM_TABLES_QUERY" 2>/dev/null) + +if [ ${#SYSTEM_TABLES[@]} -eq 0 ]; then + echo "[$SCRIPT_NAME] Error: Failed to retrieve system tables list or no tables found." + exit 1 +fi + +echo "[$SCRIPT_NAME] Found ${#SYSTEM_TABLES[@]} system tables to process." + +# --- Auto-generate System Tables Documentation --- +echo "[$SCRIPT_NAME] Auto-generating system tables documentation..." + +# Generate system tables documentation using the consolidated SQL file +SYSTEM_TABLES_SQL_FILE="$SCRIPT_DIR/system-tables.sql" +if [ -f "$SYSTEM_TABLES_SQL_FILE" ]; then + echo "[$SCRIPT_NAME] Generating system tables documentation using consolidated SQL file..." + + for TABLE in "${SYSTEM_TABLES[@]}"; do + echo "[$SCRIPT_NAME] Processing system table: $TABLE" + "$script_path" --param_table="$TABLE" --queries-file "$SYSTEM_TABLES_SQL_FILE" || { + echo "[$SCRIPT_NAME] FATAL: Failed to execute queries for system.$TABLE table."; + echo "[$SCRIPT_NAME] Attempting to re-run with output:" + "$script_path" --param_table="$TABLE" --queries-file "$SYSTEM_TABLES_SQL_FILE" + exit 1; + } + + # Rename the temporary output file to the correct name + if [ -f "temp-system-table.md" ]; then + mv "temp-system-table.md" "${TABLE}.md" || { + echo "[$SCRIPT_NAME] Error: Failed to rename temp-system-table.md to ${TABLE}.md" + exit 1 + } + echo "[$SCRIPT_NAME] Generated ${TABLE}.md" + else + echo "[$SCRIPT_NAME] Warning: temp-system-table.md not found for system.$TABLE" + fi + done +else + echo "[$SCRIPT_NAME] Warning: System tables SQL file not found at $SYSTEM_TABLES_SQL_FILE" + echo "[$SCRIPT_NAME] Skipping system tables generation." +fi + +# --- Insert generated content between AUTOGENERATED tags --- +echo "[$SCRIPT_NAME] Inserting system tables content between AUTOGENERATED_START and AUTOGENERATED_END tags" + +for TABLE in "${SYSTEM_TABLES[@]}"; do + src_file="${TABLE}.md" + # Construct full destination path using $target_dir (project root) + dest_full_path="$target_dir/docs/operations/system-tables/${TABLE}.md" + # Source file is relative to CWD ($tmp_dir) + src_rel_path="./$src_file" + + if [ -f "$src_rel_path" ]; then + # Ensure destination directory exists + mkdir -p "$(dirname "$dest_full_path")" || { + echo "[$SCRIPT_NAME] Error: Failed to create directory for $dest_full_path"; + exit 1; + } + + # Check if destination file exists and has AUTOGENERATED tags + if [ -f "$dest_full_path" ] && grep -q "" "$dest_full_path" && grep -q "" "$dest_full_path"; then + echo "[$SCRIPT_NAME] Updating content in existing file: $dest_full_path" + insert_content_between_tags "$src_rel_path" "$dest_full_path" + else + echo "[$SCRIPT_NAME] Warning: File $dest_full_path does not exist or missing AUTOGENERATED tags. Skipping." + echo "[$SCRIPT_NAME] You may need to create the file with proper AUTOGENERATED_START and AUTOGENERATED_END tags." + fi + + # Remove source file after processing + rm -f "$src_rel_path" + else + echo "[$SCRIPT_NAME] Warning: Expected output file $src_rel_path not found in $tmp_dir. Skipping." + fi +done + +echo "[$SCRIPT_NAME] System tables documentation completed." +echo "[$SCRIPT_NAME] Processed ${#SYSTEM_TABLES[@]} system tables." + # --- Final Cleanup --- echo "[$SCRIPT_NAME] Performing final cleanup..." # Determine parent dir of tmp_dir for cd step @@ -487,5 +533,5 @@ echo "[$SCRIPT_NAME] Removing contents of temporary directory: $tmp_dir" rm -rf "$tmp_dir"/* rmdir "$tmp_dir" 2>/dev/null || echo "[$SCRIPT_NAME] Unable to remove $tmp_dir" -echo "[$SCRIPT_NAME] Auto-generation of settings markdown pages completed successfully." -exit 0 +echo "[$SCRIPT_NAME] Auto-generation of settings and system tables documentation completed successfully." +exit 0 \ No newline at end of file diff --git a/scripts/settings/system-tables.sql b/scripts/settings/system-tables.sql new file mode 100644 index 00000000000..c8cd83294c1 --- /dev/null +++ b/scripts/settings/system-tables.sql @@ -0,0 +1,52 @@ +-- Consolidated system table documentation generator with nested enum handling +WITH table_columns AS ( + SELECT + name, + type, + comment, + position + FROM system.columns + WHERE database = 'system' AND table = {table:String} + ORDER BY position +), +formatted_columns AS ( + SELECT + CASE + -- Check if this is an Enum type with possible values listed + WHEN startsWith(type, 'Enum') AND position(comment, 'Possible values:') > 0 THEN + concat( + '- `', name, '` ([', type, '](../../sql-reference/data-types/)) — ', + -- Main description before "Possible values:" + trim(substring(comment, 1, position(comment, 'Possible values:') - 1)), + 'Possible values:\n', + -- Parse and format each enum value as nested bullet + arrayStringConcat( + arrayMap( + x -> concat(' - **', + trim(replaceRegexpOne(splitByString(' — ', x)[1], '^[''"]|[''"]$', '')), + '** — ', + trim(arrayElement(splitByString(' — ', x), 2))), + arrayFilter( + x -> length(trim(x)) > 0, + splitByString(', ', + trim(substring(comment, position(comment, 'Possible values:') + length('Possible values:')))) + ) + ), + char(10) + ) + ) + -- Regular column (non-enum or enum without parsed values) + ELSE + concat('- `', name, '` ([', type, '](../../sql-reference/data-types/)) — ', comment) + END as formatted_column + FROM table_columns + WHERE comment != '' +) +SELECT + arrayStringConcat( + groupArray(formatted_column), + char(10) + ) +FROM formatted_columns +LIMIT 1 +INTO OUTFILE 'temp-system-table.md' TRUNCATE FORMAT LineAsString \ No newline at end of file