update ch core to v25.8.2.29

Author: kafka1991

.clang-tidy

@@ -7,6 +7,9 @@
 
 HeaderFilterRegex: '^.*/(base|src|programs|utils)/.*(h|hpp)$'
 
+# We don't want to use clang-tidy diagnostics in the "contrib" folder.
+ExcludeHeaderFilterRegex: '^.*/contrib/.*'
+
 Checks: [
   '*',
 
@@ -28,6 +31,7 @@ Checks: [
   '-bugprone-unchecked-optional-access',
   '-bugprone-crtp-constructor-accessibility',
   '-bugprone-not-null-terminated-result',
+  '-bugprone-forward-declaration-namespace',
 
   '-cert-dcl16-c',
   '-cert-err58-cpp',

.github/PULL_REQUEST_TEMPLATE.md

@@ -17,7 +17,7 @@ tests/ci/cancel_and_rerun_workflow_lambda/app.py
 - Not for changelog (changelog entry is not required)
 
 
-### Changelog entry (a user-readable short description of the changes that goes to CHANGELOG.md):
+### Changelog entry (a [user-readable short description](https://github.com/ClickHouse/ClickHouse/blob/master/docs/changelog_entry_guidelines.md) of the changes that goes into CHANGELOG.md):
 ...
 
 ### Documentation entry for user-facing changes

.github/workflows/vectorsearchstress.yml

@@ -0,0 +1,128 @@
+# generated by praktika
+
+name: VectorSearchStress
+on:
+  schedule:
+    - cron: 47 15 * * 0
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}
+
+env:
+  PYTHONUNBUFFERED: 1
+  CHECKOUT_REF: ""
+
+jobs:
+
+  config_workflow:
+    runs-on: [self-hosted, style-checker-aarch64]
+    needs: []
+    name: "Config Workflow"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_vectorsearchstress.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Config Workflow' --workflow "VectorSearchStress" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Config Workflow' --workflow "VectorSearchStress" --ci |& tee ./ci/tmp/job.log
+          fi
+
+  vector_search_stress:
+    runs-on: [self-hosted, arm-small]
+    needs: [config_workflow]
+    name: "Vector Search Stress"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_vectorsearchstress.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Vector Search Stress' --workflow "VectorSearchStress" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Vector Search Stress' --workflow "VectorSearchStress" --ci |& tee ./ci/tmp/job.log
+          fi
+
+  finish_workflow:
+    runs-on: [self-hosted, style-checker-aarch64]
+    needs: [config_workflow, vector_search_stress]
+    if: ${{ !cancelled() }}
+    name: "Finish Workflow"
+    outputs:
+      data: ${{ steps.run.outputs.DATA }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ env.CHECKOUT_REF }}
+
+      - name: Prepare env script
+        run: |
+          rm -rf ./ci/tmp ./ci/tmp ./ci/tmp
+          mkdir -p ./ci/tmp ./ci/tmp ./ci/tmp
+          cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF'
+          export PYTHONPATH=./ci:.:
+          cat > ./ci/tmp/workflow_config_vectorsearchstress.json << 'EOF'
+          ${{ needs.config_workflow.outputs.data }}
+          EOF
+          cat > ./ci/tmp/workflow_status.json << 'EOF'
+          ${{ toJson(needs) }}
+          EOF
+          ENV_SETUP_SCRIPT_EOF
+
+      - name: Run
+        id: run
+        run: |
+          . ./ci/tmp/praktika_setup_env.sh
+          set -o pipefail
+          if command -v ts &> /dev/null; then
+            python3 -m praktika run 'Finish Workflow' --workflow "VectorSearchStress" --ci |& ts '[%Y-%m-%d %H:%M:%S]' | tee ./ci/tmp/job.log
+          else
+            python3 -m praktika run 'Finish Workflow' --workflow "VectorSearchStress" --ci |& tee ./ci/tmp/job.log
+          fi

.gitignore

@@ -19,6 +19,7 @@
 *.log
 *.debuglog
 *.stderr
+*.stderr-fatal*
 *.stdout
 
 # llvm-xray logs
@@ -167,6 +168,7 @@ tests/queries/0_stateless/*.binary
 tests/queries/0_stateless/*.generated-expect
 tests/queries/0_stateless/*.expect.history
 tests/integration/**/_gen
+tests/casa_del_dolor/_instances*
 
 # pytest --pdb history
 .pdb_history

.gitmodules

@@ -40,6 +40,9 @@
 [submodule "contrib/boost"]
 	path = contrib/boost
 	url = https://github.com/ClickHouse/boost
+[submodule "contrib/arrow"]
+	path = contrib/arrow
+	url = https://github.com/ClickHouse/arrow
 [submodule "contrib/thrift"]
 	path = contrib/thrift
 	url = https://github.com/apache/thrift
@@ -134,9 +137,6 @@
 [submodule "contrib/fmtlib"]
 	path = contrib/fmtlib
 	url = https://github.com/fmtlib/fmt
-[submodule "contrib/sentry-native"]
-	path = contrib/sentry-native
-	url = https://github.com/ClickHouse/sentry-native
 [submodule "contrib/krb5"]
 	path = contrib/krb5
 	url = https://github.com/ClickHouse/krb5
@@ -296,9 +296,6 @@
 [submodule "contrib/aws-c-compression"]
 	path = contrib/aws-c-compression
 	url = https://github.com/awslabs/aws-c-compression
-[submodule "contrib/aws-s2n-tls"]
-	path = contrib/aws-s2n-tls
-	url = https://github.com/ClickHouse/s2n-tls
 [submodule "contrib/crc32-vpmsum"]
 	path = contrib/crc32-vpmsum
 	url = https://github.com/antonblanchard/crc32-vpmsum.git
@@ -372,9 +369,27 @@
 [submodule "contrib/SHA3IUF"]
 	path = contrib/SHA3IUF
 	url = https://github.com/brainhub/SHA3IUF.git
-[submodule "contrib/arrow"]
-	path = contrib/arrow
-	url = https://github.com/chdb-io/arrow.git
+[submodule "contrib/chdig"]
+	path = contrib/chdig
+	url = https://github.com/azat/chdig.git
+[submodule "contrib/ai-sdk-cpp"]
+	path = contrib/ai-sdk-cpp
+	url = https://github.com/ClickHouse/ai-sdk-cpp
+[submodule "contrib/nlohmann-json"]
+	path = contrib/nlohmann-json
+	url = https://github.com/nlohmann/json.git
+[submodule "contrib/crc32c"]
+	path = contrib/crc32c
+	url = https://github.com/google/crc32c.git
+[submodule "contrib/simde"]
+	path = contrib/simde
+	url = https://github.com/simd-everywhere/simde
+[submodule "contrib/FastPFOR"]
+	path = contrib/FastPFOR
+	url = https://github.com/fast-pack/FastPFOR
+[submodule "contrib/antlr4-cpp-runtime"]
+	path = contrib/antlr4-cpp-runtime
+	url = https://github.com/antlr/antlr4.git
 [submodule "contrib/utf8proc"]
 	path = contrib/utf8proc
 	url = https://github.com/JuliaStrings/utf8proc.git

.snyk

@@ -234,6 +234,8 @@ if (SPLIT_DEBUG_SYMBOLS)
     set(SPLIT_DEBUG_SYMBOLS_DIR "stripped" CACHE STRING "A separate directory for stripped information")
 endif()
 
+option(BUILD_STRIPPED_BINARY "Build stripped binary (clickhouse-stripped)" OFF)
+
 cmake_host_system_information(RESULT AVAILABLE_PHYSICAL_MEMORY QUERY AVAILABLE_PHYSICAL_MEMORY) # Not available under freebsd
 
 
@@ -302,9 +304,8 @@ set (CMAKE_C_STANDARD_REQUIRED ON)
 # See https://reviews.llvm.org/D112921
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsized-deallocation")
 
-# falign-functions=32 prevents from random performance regressions with the code change. Thus, providing more stable
-# benchmarks.
-set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=32")
+# falign-functions=64 prevents from random performance regressions with the code change. Thus, providing more stable benchmarks.
+set(COMPILER_FLAGS "${COMPILER_FLAGS} -falign-functions=64")
 
 if (ARCH_AMD64)
     # align branches within a 32-Byte boundary to avoid the potential performance loss when code layout change,
@@ -381,6 +382,15 @@ elseif (ENABLE_THINLTO)
     message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot enable ThinLTO")
 endif ()
 
+if(COMPILER_PIPE)
+    set(MAX_COMPILER_MEMORY 2500)
+else()
+    set(MAX_COMPILER_MEMORY 1500)
+endif()
+set(MAX_LINKER_MEMORY 5000)
+include(cmake/limit_jobs.cmake)
+
+
 # Turns on all external libs like s3, kafka, ODBC, ...
 option(ENABLE_LIBRARIES "Enable all external libraries by default" ON)
 
@@ -402,15 +412,6 @@ endif ()
 link_libraries(global-group)
 target_link_libraries(global-group INTERFACE $<TARGET_PROPERTY:global-libs,INTERFACE_LINK_LIBRARIES>)
 
-option (ENABLE_GWP_ASAN "Enable Gwp-Asan" OFF)
-# We use mmap for allocations more heavily in debug builds,
-# but GWP-ASan also wants to use mmap frequently,
-# and due to a large number of memory mappings,
-# it does not work together well.
-# if ((NOT OS_LINUX AND NOT OS_ANDROID) OR (CMAKE_BUILD_TYPE_UC STREQUAL "DEBUG"))
-#     set(ENABLE_GWP_ASAN OFF)
-# endif ()
-
 option (ENABLE_FIU "Enable Fiu" ON)
 
 option(WERROR "Enable -Werror compiler option" ON)
@@ -517,13 +518,32 @@ macro (clickhouse_add_executable target)
         add_executable (${ARGV} $<TARGET_OBJECTS:clickhouse_malloc>)
     endif ()
 
+    # Wrap the malloc/free and other C-style functions with our own ones
+    # to inject memory tracking mechanism into them.
+    # Sanitizers have their own way of intercepting the
+    # allocations and deallocations, so we skip this step for them.
+    if (NOT (SANITIZE OR SANITIZE_COVERAGE OR OS_DARWIN OR OS_FREEBSD))
+        target_link_options(${target} PRIVATE
+            "LINKER:--wrap=malloc"
+            "LINKER:--wrap=free"
+            "LINKER:--wrap=calloc"
+            "LINKER:--wrap=realloc"
+            "LINKER:--wrap=aligned_alloc"
+            "LINKER:--wrap=posix_memalign"
+            "LINKER:--wrap=valloc"
+            "LINKER:--wrap=memalign"
+            "LINKER:--wrap=reallocarray"
+        )
+        if (NOT USE_MUSL)
+            target_link_options(${target} PRIVATE
+                "LINKER:--wrap=pvalloc"
+            )
+        endif()
+    endif()
+
     get_target_property (type ${target} TYPE)
     if (${type} STREQUAL EXECUTABLE)
-        # Disabled if memory tracking is disabled
-        if (TARGET clickhouse_new_delete)
-            # operator::new/delete for executables (MemoryTracker stuff)
-            target_link_libraries (${target} PRIVATE clickhouse_new_delete)
-        endif()
+        target_link_libraries (${target} PUBLIC clickhouse_new_delete)
 
         # In case of static jemalloc, because zone_register() is located in zone.c and
         # is never used outside (it is declared as constructor) it is omitted
@@ -595,6 +615,9 @@ option(CHECK_LARGE_OBJECT_SIZES "Check that there are no large object files afte
 
 add_subdirectory (base)
 add_subdirectory (src)
+if (LEXER_STANDALONE_BUILD)
+    add_subdirectory(tests/lexer)
+endif ()
 add_subdirectory (programs)
 add_subdirectory (utils)
 

CMakeLists.txt

@@ -14,9 +14,12 @@ The following versions of ClickHouse server are currently supported with securit
 
 | Version | Supported |
 |:-|:-|
-| 25.4 | ✔️ |
+| 25.7 | ✔️ |
+| 25.6 | ✔️ |
+| 25.5 | ✔️ |
+| 25.4 | ❌ |
 | 25.3 | ✔️ |
-| 25.2 | ✔️ |
+| 25.2 | ❌ |
 | 25.1 | ❌ |
 | 24.12 | ❌ |
 | 24.11 | ❌ |

SECURITY.md

@@ -1,4 +1,5 @@
 #include <base/Decimal.h>
+#include <base/defines.h>
 #include <base/extended_types.h>
 
 namespace DB
@@ -72,13 +73,15 @@ template <typename T> Decimal<T> operator- (const Decimal<T> & x, const Decimal<
 template <typename T> Decimal<T> operator* (const Decimal<T> & x, const Decimal<T> & y) { return x.value * y.value; }
 template <typename T> Decimal<T> operator/ (const Decimal<T> & x, const Decimal<T> & y) { return x.value / y.value; }
 template <typename T> Decimal<T> operator- (const Decimal<T> & x) { return -x.value; }
+template <typename T> Decimal<T> NO_SANITIZE_UNDEFINED negateOverflow (const Decimal<T> & x) { return -x.value; }
 
 #define DISPATCH(TYPE) \
 template Decimal<TYPE> operator+ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
 template Decimal<TYPE> operator- (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
 template Decimal<TYPE> operator* (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
 template Decimal<TYPE> operator/ (const Decimal<TYPE> & x, const Decimal<TYPE> & y); \
-template Decimal<TYPE> operator- (const Decimal<TYPE> & x);
+template Decimal<TYPE> operator- (const Decimal<TYPE> & x); \
+template Decimal<TYPE> NO_SANITIZE_UNDEFINED negateOverflow (const Decimal<TYPE> & x);
 FOR_EACH_UNDERLYING_DECIMAL_TYPE(DISPATCH)
 #undef DISPATCH