diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c00436162..66d66ab148 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,41 +124,20 @@ if(CLICE_CI_ENVIRONMENT) target_compile_definitions(clice_options INTERFACE CLICE_CI_ENVIRONMENT=1) endif() -set(FBS_SCHEMA_FILE "${PROJECT_SOURCE_DIR}/src/index/schema.fbs") -set(GENERATED_HEADER "${PROJECT_BINARY_DIR}/generated/schema_generated.h") - -if(CMAKE_CROSSCOMPILING) - find_program(FLATC_EXECUTABLE flatc REQUIRED) - set(FLATC_CMD "${FLATC_EXECUTABLE}") -else() - set(FLATC_CMD "$") -endif() - -add_custom_command( - OUTPUT "${GENERATED_HEADER}" - COMMAND ${FLATC_CMD} --cpp -o "${PROJECT_BINARY_DIR}/generated" "${FBS_SCHEMA_FILE}" - DEPENDS "${FBS_SCHEMA_FILE}" - COMMENT "Generating C++ header from ${FBS_SCHEMA_FILE}" -) - -add_custom_target(generate_flatbuffers_schema DEPENDS "${GENERATED_HEADER}") - file(GLOB_RECURSE CLICE_CORE_SOURCES CONFIGURE_DEPENDS "${PROJECT_SOURCE_DIR}/src/*.cpp") add_library(clice-core STATIC ${CLICE_CORE_SOURCES}) add_library(clice::core ALIAS clice-core) -add_dependencies(clice-core generate_flatbuffers_schema) target_include_directories(clice-core PUBLIC "${PROJECT_SOURCE_DIR}/src" - "${PROJECT_BINARY_DIR}/generated" ) target_link_libraries(clice-core PUBLIC clice_options llvm-libs spdlog::spdlog roaring::roaring - flatbuffers kota::ipc::lsp + kota::codec::flatbuffers kota::codec::toml kota::option simdjson::simdjson diff --git a/cmake/package.cmake b/cmake/package.cmake index cd1a9f5dab..a4d5c7a48e 100644 --- a/cmake/package.cmake +++ b/cmake/package.cmake @@ -27,21 +27,10 @@ FetchContent_Declare( set(ENABLE_ROARING_TESTS OFF CACHE INTERNAL "" FORCE) set(ENABLE_ROARING_MICROBENCHMARKS OFF CACHE INTERNAL "" FORCE) -# flatbuffers -FetchContent_Declare( - flatbuffers - GIT_REPOSITORY https://github.com/google/flatbuffers.git - GIT_TAG v25.9.23 - GIT_SHALLOW TRUE -) -set(FLATBUFFERS_BUILD_GRPC OFF CACHE BOOL "" FORCE) -set(FLATBUFFERS_BUILD_TESTS OFF CACHE BOOL "" FORCE) -set(FLATBUFFERS_BUILD_FLATHASH OFF CACHE BOOL "" FORCE) - FetchContent_Declare( kotatsu GIT_REPOSITORY https://github.com/clice-io/kotatsu - GIT_TAG 02b255f + GIT_TAG 5ff9bf1 ) set(KOTA_ENABLE_ZEST ON) @@ -49,7 +38,9 @@ set(KOTA_ENABLE_TEST OFF) set(KOTA_CODEC_ENABLE_SIMDJSON ON) set(KOTA_CODEC_ENABLE_YYJSON ON) set(KOTA_CODEC_ENABLE_TOML ON) +# flatbuffers itself now comes transitively through kota::codec::flatbuffers. +set(KOTA_CODEC_ENABLE_FLATBUFFERS ON) set(KOTA_ENABLE_EXCEPTIONS OFF) set(KOTA_ENABLE_RTTI OFF) -FetchContent_MakeAvailable(kotatsu spdlog croaring flatbuffers) +FetchContent_MakeAvailable(kotatsu spdlog croaring) diff --git a/pixi.lock b/pixi.lock index 3ce32b8b57..f9bbbf27cb 100644 --- a/pixi.lock +++ b/pixi.lock @@ -53,7 +53,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/clangxx_impl_linux-64-20.1.8-default_cfg_h027053c_14.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.3.1-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/compiler-rt-20.1.8-hb700be7_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/flatbuffers-25.9.23-hb7d4c21_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc-14.2.0-h96c4ede_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-14.2.0-hdb7739f_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-aarch64-14.2.0-h0367a0d_2.conda @@ -137,7 +136,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/clangxx_impl_linux-aarch64-20.1.8-default_cfg_h02d8f17_14.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.3.1-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/compiler-rt-20.1.8-hfefdfc9_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/flatbuffers-25.9.23-h5248ec3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.22.2-hfd895c2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45.1-default_h1979696_102.conda @@ -209,7 +207,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/clangxx_impl_osx-64-20.1.8-default_cfg_h91eacd1_14.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cmake-4.3.1-h2426fb6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/compiler-rt-20.1.8-he914875_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/flatbuffers-25.9.23-h6982a40_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.3-h25d91c4_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/krb5-1.22.2-h207b36a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ld64-956.6-llvm20_1_h2b71b23_4.conda @@ -275,7 +272,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clangxx_impl_osx-arm64-20.1.8-default_cfg_h6e044b8_14.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cmake-4.3.1-h8cb302d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/compiler-rt-20.1.8-h855ad52_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/flatbuffers-25.9.23-h9e8ef45_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-hef89b57_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.22.2-h385eeb1_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ld64-956.6-llvm20_1_hb625feb_4.conda @@ -335,7 +331,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/clangxx-20.1.8-default_nocfg_hbb9487a_14.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.3.1-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/compiler-rt-20.1.8-h49e36cd_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/flatbuffers-25.9.23-h4025804_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.22.2-h0ea6238_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libclang13-22.1.3-default_ha2db4b5_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libcurl-8.19.0-h8206538_0.conda @@ -386,7 +381,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/clangxx_impl_linux-64-20.1.8-default_cfg_h027053c_14.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.3.1-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/compiler-rt-20.1.8-hb700be7_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/flatbuffers-25.9.23-hb7d4c21_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc-14.2.0-h96c4ede_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-14.2.0-hdb7739f_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gxx-14.2.0-h96c4ede_2.conda @@ -461,7 +455,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/clangxx_impl_linux-aarch64-20.1.8-default_cfg_h02d8f17_14.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.3.1-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/compiler-rt-20.1.8-hfefdfc9_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/flatbuffers-25.9.23-h5248ec3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.22.2-hfd895c2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45.1-default_h1979696_102.conda @@ -533,7 +526,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/clangxx_impl_osx-64-20.1.8-default_cfg_h91eacd1_14.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cmake-4.3.1-h2426fb6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/compiler-rt-20.1.8-he914875_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/flatbuffers-25.9.23-h6982a40_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.3-h25d91c4_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/krb5-1.22.2-h207b36a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ld64-956.6-llvm20_1_h2b71b23_4.conda @@ -599,7 +591,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clangxx_impl_osx-arm64-20.1.8-default_cfg_h6e044b8_14.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cmake-4.3.1-h8cb302d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/compiler-rt-20.1.8-h855ad52_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/flatbuffers-25.9.23-h9e8ef45_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-hef89b57_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.22.2-h385eeb1_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ld64-956.6-llvm20_1_hb625feb_4.conda @@ -659,7 +650,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/clangxx-20.1.8-default_nocfg_hbb9487a_14.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.3.1-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/compiler-rt-20.1.8-h49e36cd_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/flatbuffers-25.9.23-h4025804_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.22.2-h0ea6238_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libclang13-22.1.3-default_ha2db4b5_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libcurl-8.19.0-h8206538_0.conda @@ -710,7 +700,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/clangxx_impl_linux-64-20.1.8-default_cfg_h027053c_14.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.3.1-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/compiler-rt-20.1.8-hb700be7_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/flatbuffers-25.9.23-hb7d4c21_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc-14.2.0-h96c4ede_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-14.2.0-hdb7739f_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gxx-14.2.0-h96c4ede_2.conda @@ -785,7 +774,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/clangxx_impl_linux-aarch64-20.1.8-default_cfg_h02d8f17_14.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.3.1-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/compiler-rt-20.1.8-hfefdfc9_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/flatbuffers-25.9.23-h5248ec3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.22.2-hfd895c2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45.1-default_h1979696_102.conda @@ -857,7 +845,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/clangxx_impl_osx-64-20.1.8-default_cfg_h91eacd1_14.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cmake-4.3.1-h2426fb6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/compiler-rt-20.1.8-he914875_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/flatbuffers-25.9.23-h6982a40_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.3-h25d91c4_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/krb5-1.22.2-h207b36a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ld64-956.6-llvm20_1_h2b71b23_4.conda @@ -923,7 +910,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clangxx_impl_osx-arm64-20.1.8-default_cfg_h6e044b8_14.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cmake-4.3.1-h8cb302d_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/compiler-rt-20.1.8-h855ad52_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/flatbuffers-25.9.23-h9e8ef45_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.3-hef89b57_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.22.2-h385eeb1_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/ld64-956.6-llvm20_1_hb625feb_4.conda @@ -983,7 +969,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/clangxx-20.1.8-default_nocfg_hbb9487a_14.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.3.1-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/compiler-rt-20.1.8-h49e36cd_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/flatbuffers-25.9.23-h4025804_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.22.2-h0ea6238_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libclang13-22.1.3-default_ha2db4b5_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libcurl-8.19.0-h8206538_0.conda @@ -1035,7 +1020,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/clangxx-20.1.8-default_h363a0c9_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.2.1-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/compiler-rt-20.1.8-hb700be7_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/flatbuffers-25.9.23-hb7d4c21_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc-14.2.0-h96c4ede_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-14.2.0-hdb7739f_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gxx-14.2.0-h96c4ede_2.conda @@ -1118,7 +1102,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/clangxx_impl_linux-aarch64-20.1.8-default_cfg_h02d8f17_14.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.3.1-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/compiler-rt-20.1.8-hfefdfc9_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/flatbuffers-25.9.23-h5248ec3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.22.2-hfd895c2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45.1-default_h1979696_102.conda @@ -1198,7 +1181,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/clangxx_impl_osx-64-20.1.8-default_cfg_h91eacd1_14.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cmake-4.3.1-h2426fb6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/compiler-rt-20.1.8-he914875_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/flatbuffers-25.9.23-h6982a40_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.3-h25d91c4_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/krb5-1.22.2-h207b36a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ld64-956.6-llvm20_1_h2b71b23_4.conda @@ -1268,7 +1250,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clangxx-20.1.8-default_h36137df_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cmake-4.2.1-h54ad630_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/compiler-rt-20.1.8-h855ad52_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/flatbuffers-25.9.23-h9e8ef45_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-78.1-h38cb7af_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.21.3-h237132a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libclang-cpp20.1-20.1.8-default_h73dfc95_5.conda @@ -1330,7 +1311,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/clangxx-20.1.8-default_hac490eb_5.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.2.1-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/compiler-rt-20.1.8-h49e36cd_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/flatbuffers-25.9.23-h4025804_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/icu-78.1-h637d24d_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libclang13-22.1.0-default_ha2db4b5_0.conda @@ -1789,7 +1769,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-64/clangxx-20.1.8-default_h363a0c9_5.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/cmake-4.2.1-hc85cc9f_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/compiler-rt-20.1.8-hb700be7_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-64/flatbuffers-25.9.23-hb7d4c21_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc-14.2.0-h96c4ede_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc_impl_linux-64-14.2.0-hdb7739f_2.conda - conda: https://conda.anaconda.org/conda-forge/linux-64/gxx-14.2.0-h96c4ede_2.conda @@ -1875,7 +1854,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/clangxx_impl_linux-aarch64-20.1.8-default_cfg_h02d8f17_14.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/cmake-4.3.1-hc9d863e_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/compiler-rt-20.1.8-hfefdfc9_1.conda - - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/flatbuffers-25.9.23-h5248ec3_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/keyutils-1.6.3-h86ecc28_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/krb5-1.22.2-hfd895c2_0.conda - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/ld_impl_linux-aarch64-2.45.1-default_h1979696_102.conda @@ -1959,7 +1937,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-64/clangxx_impl_osx-64-20.1.8-default_cfg_h91eacd1_14.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/cmake-4.3.1-h2426fb6_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/compiler-rt-20.1.8-he914875_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-64/flatbuffers-25.9.23-h6982a40_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.3-h25d91c4_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/krb5-1.22.2-h207b36a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-64/ld64-956.6-llvm20_1_h2b71b23_4.conda @@ -2033,7 +2010,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/osx-arm64/clangxx-20.1.8-default_h36137df_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/cmake-4.2.1-h54ad630_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/compiler-rt-20.1.8-h855ad52_1.conda - - conda: https://conda.anaconda.org/conda-forge/osx-arm64/flatbuffers-25.9.23-h9e8ef45_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/krb5-1.21.3-h237132a_0.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libclang-cpp20.1-20.1.8-default_h73dfc95_5.conda - conda: https://conda.anaconda.org/conda-forge/osx-arm64/libclang13-21.1.7-default_h13b06bd_4.conda @@ -2098,7 +2074,6 @@ environments: - conda: https://conda.anaconda.org/conda-forge/win-64/clangxx-20.1.8-default_hac490eb_5.conda - conda: https://conda.anaconda.org/conda-forge/win-64/cmake-4.2.1-hdcbee5b_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/compiler-rt-20.1.8-h49e36cd_1.conda - - conda: https://conda.anaconda.org/conda-forge/win-64/flatbuffers-25.9.23-h4025804_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/krb5-1.21.3-hdf4eb48_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libclang13-22.1.0-default_ha2db4b5_0.conda - conda: https://conda.anaconda.org/conda-forge/win-64/libcurl-8.17.0-h43ecb02_1.conda @@ -2787,18 +2762,6 @@ packages: license_family: MIT size: 1209421 timestamp: 1757336717570 -- conda: https://conda.anaconda.org/conda-forge/linux-64/flatbuffers-25.9.23-hb7d4c21_0.conda - sha256: e5f90c2fd61012d6ad332657a5bf5455620f0db8524f0b005d91e1c2737bad69 - md5: 10a330bfd5345af730b0fc1349d15eaf - depends: - - __glibc >=2.17,<3.0.a0 - - libgcc >=14 - - libstdcxx >=14 - license: Apache-2.0 - license_family: APACHE - purls: [] - size: 1584732 - timestamp: 1761142459651 - conda: https://conda.anaconda.org/conda-forge/linux-64/gcc-14.2.0-h96c4ede_2.conda sha256: 6592ff15983759799df89d0a992268809e9ce0376933e053716660ecda19d87e md5: 7f23fa6a103277a8491d3d10240ebd09 @@ -4646,17 +4609,6 @@ packages: license_family: MIT size: 1113801 timestamp: 1773352768018 -- conda: https://conda.anaconda.org/conda-forge/linux-aarch64/flatbuffers-25.9.23-h5248ec3_0.conda - sha256: 9e0559987ca6a4a5b4a942514709311993ded4b1fb911a1fc69876ba3e221fe0 - md5: b68619170886a3289fdc167e1120122c - depends: - - libgcc >=14 - - libstdcxx >=14 - license: Apache-2.0 - license_family: APACHE - purls: [] - size: 1414741 - timestamp: 1761328999536 - conda: https://conda.anaconda.org/conda-forge/linux-aarch64/icu-78.3-hcab7f73_0.conda sha256: 49ba6aed2c6b482bb0ba41078057555d29764299bc947b990708617712ef6406 md5: 546da38c2fa9efacf203e2ad3f987c59 @@ -6073,17 +6025,6 @@ packages: license_family: MIT size: 1133353 timestamp: 1773353161332 -- conda: https://conda.anaconda.org/conda-forge/osx-64/flatbuffers-25.9.23-h6982a40_0.conda - sha256: 7d430e5fbe47e690e00345e4c5a856d2469aa1ea6afc573e8dabd0eb8aabea5f - md5: 314a166f491f531788ad6220acad496c - depends: - - __osx >=10.13 - - libcxx >=19 - license: Apache-2.0 - license_family: APACHE - purls: [] - size: 1358478 - timestamp: 1761142801269 - conda: https://conda.anaconda.org/conda-forge/osx-64/icu-78.3-h25d91c4_0.conda sha256: 1294117122d55246bb83ad5b589e2a031aacdf2d0b1f99fd338aa4394f881735 md5: 627eca44e62e2b665eeec57a984a7f00 @@ -7221,17 +7162,6 @@ packages: license_family: MIT size: 1050638 timestamp: 1757337263602 -- conda: https://conda.anaconda.org/conda-forge/osx-arm64/flatbuffers-25.9.23-h9e8ef45_0.conda - sha256: b8f4ce2919f2542c6688af909c18f9672b2a19efdb57118c5f415dd5ff0fb3cd - md5: 1d6e0829bc8d6907fae9a81f169414ce - depends: - - __osx >=11.0 - - libcxx >=19 - license: Apache-2.0 - license_family: APACHE - purls: [] - size: 1299156 - timestamp: 1761143339517 - conda: https://conda.anaconda.org/conda-forge/osx-arm64/icu-75.1-hfee45f7_0.conda sha256: 9ba12c93406f3df5ab0a43db8a4b4ef67a5871dfd401010fbe29b218b2cbe620 md5: 5eb22c1d7b3fc4abb50d92d621583137 @@ -8736,18 +8666,6 @@ packages: license_family: MIT size: 1196708 timestamp: 1757337405047 -- conda: https://conda.anaconda.org/conda-forge/win-64/flatbuffers-25.9.23-h4025804_0.conda - sha256: dc5a1b8bf5c667f5d86bdafd56faabbd2cfd3d498d612f709e6bf493a52c4713 - md5: e9eda0b0b9dd959828fa35e84cc09669 - depends: - - ucrt >=10.0.20348.0 - - vc >=14.3,<15 - - vc14_runtime >=14.44.35208 - license: Apache-2.0 - license_family: APACHE - purls: [] - size: 1777143 - timestamp: 1761142523983 - conda: https://conda.anaconda.org/conda-forge/win-64/icu-78.1-h637d24d_0.conda sha256: bee083d5a0f05c380fcec1f30a71ef5518b23563aeb0a21f6b60b792645f9689 md5: cb8048bed35ef01431184d6a88e46b3e diff --git a/pixi.toml b/pixi.toml index 69bbd795f3..71c14c5a05 100644 --- a/pixi.toml +++ b/pixi.toml @@ -43,7 +43,6 @@ lld = "==20.1.8" llvm-tools = "==20.1.8" clang-tools = "==20.1.8" compiler-rt = "==20.1.8" -flatbuffers = "==25.9.23" [feature.build.target.win-64.dependencies] sccache = "*" diff --git a/src/index/include_graph.h b/src/index/include_graph.h index 8e9da218c9..b63903b460 100644 --- a/src/index/include_graph.h +++ b/src/index/include_graph.h @@ -2,11 +2,13 @@ #include #include +#include #include #include #include "syntax/token.h" +#include "kota/meta/annotation.h" #include "llvm/ADT/DenseMap.h" namespace clice { @@ -41,8 +43,10 @@ struct IncludeGraph { /// Each `FileID` represents a new header context and is introduced /// by a new include directive. So a include directive is a new header - /// context. A map between FileID and its include location. - llvm::DenseMap file_table; + /// context. A map between FileID and its include location. Build-time + /// only: FileIDs are meaningless outside the producing compilation, so + /// the field is skipped during serialization. + kota::meta::skip> file_table; static IncludeGraph from(CompilationUnitRef unit); @@ -65,6 +69,17 @@ struct IncludeGraph { return paths.size() - 1; } } + + /// First include location that introduces `path_id`, or nullopt if the + /// path is never included (e.g. the main file). + std::optional first_include_of(std::uint32_t path_id) const { + for(std::uint32_t i = 0; i < locations.size(); ++i) { + if(locations[i].path_id == path_id) { + return i; + } + } + return std::nullopt; + } }; } // namespace clice::index diff --git a/src/index/merged_index.cpp b/src/index/merged_index.cpp index da4e37db80..fc3ba982fe 100644 --- a/src/index/merged_index.cpp +++ b/src/index/merged_index.cpp @@ -80,7 +80,6 @@ struct DenseMapInfo { namespace clice::index { /// (path_id, content_hash) captured for one dependency at index-build time. -/// Layout must mirror `binary::DepHash` so `safe_cast` can alias it. struct DepHash { std::uint32_t path_id; std::uint64_t content_hash; @@ -167,6 +166,10 @@ struct CompilationContext { }; struct MergedIndex::Impl { + /// On-disk schema version, serialized first so loaders can reject + /// incompatible shards before touching anything else. + std::uint32_t format_version = index_format_version; + /// Shard-local path table: every path id stored in this shard indexes /// into it, so shards are self-contained across sessions (runtime pool /// ids never persist). @@ -195,8 +198,9 @@ struct MergedIndex::Impl { /// The max canonical id we have allocated. std::uint32_t max_canonical_id = 0; - /// The reference count of each canonical id. - std::vector canonical_ref_counts; + /// The reference count of each canonical id. Derived state: rebuilt from + /// the contexts on load, never serialized. + kota::meta::skip> canonical_ref_counts; /// The canonical id set of removed index. roaring::Roaring removed; @@ -210,8 +214,9 @@ struct MergedIndex::Impl { /// Symbols local to this file (FileLocal) or TU (TULocal). SymbolTable symbols; - /// Sorted occurrences cache for fast lookup. - std::vector occurrences_cache; + /// Sorted occurrences cache for fast lookup. Derived state, never + /// serialized. + kota::meta::skip> occurrences_cache; /// Drop one reference to a canonical index; the last reference masks its /// occurrences and relations via the removed bitmap. A later re-merge of @@ -253,9 +258,68 @@ struct MergedIndex::Impl { self.max_canonical_id += 1; } + /// Vacuum state masked by the removed bitmap so the reflected object can + /// be serialized as-is: rows whose every canonical was released are + /// dropped, live bitmaps are pre-subtracted, dead cache entries go with + /// them (a later re-merge of identical content mints a fresh canonical), + /// and the removed bitmap ends up empty. This matches exactly what a + /// load of the compacted bytes would reconstruct. + void compact(this Impl& self) { + self.occurrences_cache.clear(); + + auto& removed = self.removed; + if(removed.isEmpty()) { + return; + } + + llvm::SmallVector> dead_hashes; + for(auto& entry: self.canonical_cache) { + if(removed.contains(entry.second)) { + dead_hashes.emplace_back(entry.first()); + } + } + for(auto& hash: dead_hashes) { + self.canonical_cache.erase(hash); + } + + llvm::DenseMap live_occurrences; + for(auto& [occurrence, bitmap]: self.occurrences) { + auto masked = bitmap - removed; + if(!masked.isEmpty()) { + live_occurrences.try_emplace(occurrence, std::move(masked)); + } + } + self.occurrences = std::move(live_occurrences); + + llvm::DenseMap> live_relations; + for(auto& [symbol_id, symbol_relations]: self.relations) { + llvm::DenseMap live; + for(auto& [relation, bitmap]: symbol_relations) { + auto masked = bitmap - removed; + if(!masked.isEmpty()) { + live.try_emplace(relation, std::move(masked)); + } + } + if(!live.empty()) { + live_relations.try_emplace(symbol_id, std::move(live)); + } + } + self.relations = std::move(live_relations); + + removed = roaring::Roaring(); + } + friend bool operator==(const Impl&, const Impl&) = default; }; +namespace { + +std::span buffer_bytes(const llvm::MemoryBuffer& buffer) { + return {reinterpret_cast(buffer.getBufferStart()), buffer.getBufferSize()}; +} + +} // namespace + MergedIndex::MergedIndex(std::unique_ptr buffer, std::unique_ptr impl) : buffer(std::move(buffer)), impl(std::move(impl)) {} @@ -281,97 +345,35 @@ void MergedIndex::load_in_memory(this Self& self) { } auto& index = *self.impl; - auto root = fbs::GetRoot(self.buffer->getBufferStart()); - index.max_canonical_id = root->max_canonical_id(); - - if(root->paths()) { - for(auto path: *root->paths()) { - index.paths.path_id(path->string_view()); - } - } - - for(auto entry: *root->canonical_cache()) { - index.canonical_cache.try_emplace(entry->sha256()->string_view(), entry->canonical_id()); + if(!read_flatbuffer(self.buffer->getBufferStart(), self.buffer->getBufferSize(), index)) { + index = Impl(); + self.buffer.reset(); + return; } + // Rebuild the derived reference counts from the decoded contexts. Ids are + // bounds-checked: the verifier guarantees structure, not semantics, so a + // corrupted-but-well-formed shard must not index out of range. index.canonical_ref_counts.resize(index.max_canonical_id, 0); - - for(auto entry: *root->header_contexts()) { - HeaderContext context; - auto path = entry->path_id(); - context.version = entry->version(); - for(auto include: *entry->includes()) { - index.canonical_ref_counts[include->canonical_id()] += 1; - context.includes.emplace_back(*safe_cast(include)); - } - index.header_contexts.try_emplace(path, std::move(context)); - } - - for(auto entry: *root->compilation_contexts()) { - CompilationContext context; - auto path = entry->path_id(); - context.version = entry->version(); - context.canonical_id = entry->canonical_id(); - context.build_at = entry->build_at(); - for(auto include: *entry->include_locations()) { - context.include_locations.emplace_back(*safe_cast(include)); - } - if(entry->dep_hashes()) { - for(auto dep: *entry->dep_hashes()) { - context.dep_hashes.emplace_back(*safe_cast(dep)); - } + auto count_ref = [&](std::uint32_t canonical_id) { + if(canonical_id < index.canonical_ref_counts.size()) { + index.canonical_ref_counts[canonical_id] += 1; } - index.compilation_contexts.try_emplace(path, std::move(context)); - } - - // Count ref counts from compilation contexts. - for(auto entry: *root->compilation_contexts()) { - index.canonical_ref_counts[entry->canonical_id()] += 1; - } - - // Deserialize removed bitmap. - if(root->removed() && root->removed()->size() > 0) { - index.removed = read_bitmap(root->removed()); - } - - for(auto entry: *root->occurrences()) { - index.occurrences.try_emplace(*safe_cast(entry->occurrence()), - read_bitmap(entry->context())); - } - - for(auto entry: *root->relations()) { - auto& relations = index.relations[entry->symbol()]; - for(auto relation_entry: *entry->relations()) { - relations.try_emplace(*safe_cast(relation_entry->relation()), - read_bitmap(relation_entry->context())); + }; + for(auto& [_, context]: index.header_contexts) { + for(auto& include: context.includes) { + count_ref(include.canonical_id); } } - - if(root->content()) { - index.content = root->content()->str(); + for(auto& [_, context]: index.compilation_contexts) { + count_ref(context.canonical_id); } - if(root->line_starts() && root->line_starts()->size() > 0) { - auto* ls = root->line_starts(); - index.line_starts.assign(ls->begin(), ls->end()); - } else if(!index.content.empty()) { + if(index.line_starts.empty() && !index.content.empty()) { index.line_starts = kota::ipc::lsp::build_line_starts(index.content); } - if(root->symbols()) { - for(auto entry: *root->symbols()) { - auto& symbol = index.symbols[entry->symbol_id()]; - if(auto* s = entry->symbol()) { - if(s->name()) - symbol.name = s->name()->str(); - symbol.kind = SymbolKind(static_cast(s->kind())); - symbol.scope = static_cast(s->scope()); - symbol.reference_files = read_bitmap(s->refs()); - } - } - } - self.buffer.reset(); } @@ -382,25 +384,24 @@ MergedIndex MergedIndex::load(llvm::StringRef path) { } // A stale cache directory from an older build must never crash the server - // or be misread. Verify the blob is a structurally valid flatbuffer, then - // discard any shard whose format version differs (including version-less - // shards, which report 0). A discarded shard is treated as "not on disk" - // and the background indexer rebuilds it. - auto data = reinterpret_cast((*buffer)->getBufferStart()); - fbs::Verifier verifier(data, (*buffer)->getBufferSize()); - if(!verifier.VerifyBuffer(nullptr)) { + // or be misread. Deep-verify the blob against the reflected shard schema + // (flatc-era blobs already fail the kotatsu buffer identifier check), + // then discard any shard whose format version differs. A discarded shard + // is treated as "not on disk" and the background indexer rebuilds it. + auto bytes = buffer_bytes(**buffer); + if(!kota::codec::fbs::verify_flatbuffer(bytes)) { return MergedIndex(); } - auto root = fbs::GetRoot((*buffer)->getBufferStart()); - if(root->format_version() != index_format_version) { + auto root = kota::codec::fbs::table_view::from_bytes(bytes); + if(root[&Impl::format_version] != index_format_version) { return MergedIndex(); } return MergedIndex(std::move(*buffer), nullptr); } -void MergedIndex::serialize(this const Self& self, llvm::raw_ostream& out) { +void MergedIndex::serialize(this Self& self, llvm::raw_ostream& out) { if(self.buffer) { out.write(self.buffer->getBufferStart(), self.buffer->getBufferSize()); return; @@ -410,149 +411,14 @@ void MergedIndex::serialize(this const Self& self, llvm::raw_ostream& out) { return; } - auto& index = self.impl; - - fbs::FlatBufferBuilder builder(1024); - - llvm::SmallVector buffer; - - // Compaction: rows whose every canonical was released are masked at - // runtime by the removed bitmap, but the serialized shard is served - // through buffer-only lookups that never consult it — so masked state - // must not reach disk at all. Dead rows are dropped, live bitmaps are - // written pre-subtracted, dead cache entries go with them (a later - // re-merge of identical content mints a fresh canonical), and the - // persisted removed bitmap is always empty. - auto& removed = index->removed; - auto live = [&](const roaring::Roaring& bitmap) { - return removed.isEmpty() ? bitmap : bitmap - removed; - }; - - llvm::SmallVector> canonical_cache; - for(auto& [hash, canonical_id]: index->canonical_cache) { - if(removed.contains(canonical_id)) { - continue; - } - canonical_cache.push_back( - binary::CreateCacheEntry(builder, CreateString(builder, hash.str()), canonical_id)); - } - - auto header_contexts = transform(index->header_contexts, [&](auto&& value) { - auto& [path_id, context] = value; - return binary::CreateHeaderContextEntry( - builder, - path_id, - context.version, - CreateStructVector(builder, context.includes)); - }); - - auto compilation_contexts = transform(index->compilation_contexts, [&](auto&& value) { - auto& [path_id, context] = value; - return binary::CreateCompilationContextEntry( - builder, - path_id, - context.version, - context.canonical_id, - context.build_at, - CreateStructVector(builder, context.include_locations), - CreateStructVector(builder, context.dep_hashes)); - }); - - llvm::SmallVector occurrence_keys; - llvm::SmallVector> occurrences; - occurrence_keys.reserve(index->occurrences.size()); - occurrences.reserve(index->occurrences.size()); - for(auto& [occurrence, bitmap]: index->occurrences) { - auto masked = live(bitmap); - if(masked.isEmpty()) { - continue; - } - buffer.clear(); - buffer.resize_for_overwrite(masked.getSizeInBytes(false)); - masked.write(buffer.data(), false); - occurrence_keys.emplace_back(&occurrence); - occurrences.push_back( - binary::CreateOccurrenceEntry(builder, - safe_cast(&occurrence), - CreateVector(builder, buffer))); - } - std::ranges::sort(std::views::zip(occurrence_keys, occurrences), [](auto lhs, auto rhs) { - const auto& lo = *std::get<0>(lhs); - const auto& ro = *std::get<0>(rhs); - return std::tuple(lo.range.begin, lo.range.end, lo.target) < - std::tuple(ro.range.begin, ro.range.end, ro.target); - }); - - llvm::SmallVector relation_keys; - llvm::SmallVector> relations; - relation_keys.reserve(index->relations.size()); - relations.reserve(index->relations.size()); - for(auto& [symbol_id, symbol_relations]: index->relations) { - llvm::SmallVector> entries; - for(auto& [relation, bitmap]: symbol_relations) { - auto masked = live(bitmap); - if(masked.isEmpty()) { - continue; - } - buffer.clear(); - buffer.resize_for_overwrite(masked.getSizeInBytes(false)); - masked.write(buffer.data(), false); - entries.push_back(binary::CreateRelationEntry(builder, - safe_cast(&relation), - CreateVector(builder, buffer))); - } - if(entries.empty()) { - continue; - } - relation_keys.emplace_back(symbol_id); - relations.push_back( - binary::CreateSymbolRelationsEntry(builder, symbol_id, CreateVector(builder, entries))); - } - std::ranges::sort(std::views::zip(relation_keys, relations), {}, [](auto e) { - return std::get<0>(e); - }); - - // Post-compaction nothing on disk is masked; the persisted removed - // bitmap is always empty. - buffer.clear(); - auto removed_offset = CreateVector(builder, buffer); - - auto content_offset = CreateString(builder, index->content); - auto line_starts_offset = builder.CreateVector(index->line_starts); - - auto symbols = transform(index->symbols, [&](auto&& value) { - auto& [symbol_id, symbol] = value; - buffer.clear(); - buffer.resize_for_overwrite(symbol.reference_files.getSizeInBytes(false)); - symbol.reference_files.write(buffer.data(), false); - return binary::CreateSymbolEntry(builder, - symbol_id, - binary::CreateSymbol(builder, - CreateString(builder, symbol.name), - symbol.kind.value(), - CreateVector(builder, buffer), - static_cast(symbol.scope))); - }); - - auto paths = transform(index->paths.paths, - [&](llvm::StringRef path) { return CreateString(builder, path); }); - - auto merged_index = binary::CreateMergedIndex(builder, - index->max_canonical_id, - CreateVector(builder, paths), - CreateVector(builder, canonical_cache), - CreateVector(builder, header_contexts), - CreateVector(builder, compilation_contexts), - CreateVector(builder, occurrences), - CreateVector(builder, relations), - removed_offset, - content_offset, - line_starts_offset, - CreateVector(builder, symbols), - index_format_version); - builder.Finish(merged_index); - - out.write(safe_cast(builder.GetBufferPointer()), builder.GetSize()); + // The serialized shard is served through buffer-only lookups that never + // consult the removed bitmap, so masked state must not reach disk. + // Compacting in place produces exactly the state a reload of the written + // bytes would reconstruct, after which the reflected object encodes + // as-is. Map entries are written sorted by their keys' ordering, which + // the lazy lookups binary-search. + self.impl->compact(); + write_flatbuffer(out, *self.impl); } void MergedIndex::lookup(this const Self& self, @@ -565,60 +431,45 @@ void MergedIndex::lookup(this const Self& self, for(auto& [o, _]: index.occurrences) { occurrences.emplace_back(o); } - std::ranges::sort(occurrences, [](const Occurrence& lhs, const Occurrence& rhs) { - return std::tuple(lhs.range.begin, lhs.range.end, lhs.target) < - std::tuple(rhs.range.begin, rhs.range.end, rhs.target); - }); + std::ranges::sort(occurrences); } - auto it = std::ranges::lower_bound(occurrences, offset, {}, [](index::Occurrence& o) { - return o.range.end; - }); - - while(it != occurrences.end()) { - if(it->range.contains(offset)) { - // Skip occurrences whose canonical_ids are all removed. - if(!index.removed.isEmpty()) { - auto bitmap_it = index.occurrences.find(*it); - if(bitmap_it != index.occurrences.end()) { - auto remaining = bitmap_it->second - index.removed; - if(remaining.isEmpty()) { - it++; - continue; - } - } - } - - if(!callback(*it)) { - break; + lookup_occurrences(occurrences, offset, [&](const Occurrence& occurrence) { + // Skip occurrences whose canonical_ids are all removed. + if(!index.removed.isEmpty()) { + auto bitmap_it = index.occurrences.find(occurrence); + if(bitmap_it != index.occurrences.end() && + (bitmap_it->second - index.removed).isEmpty()) { + return true; } - - it++; - continue; } - - break; - } - } else if(self.buffer) { - auto index = fbs::GetRoot(self.buffer->getBufferStart()); - auto& occurrences = *index->occurrences(); - - auto it = std::ranges::lower_bound(occurrences, offset, {}, [](auto o) { - return o->occurrence()->range().end(); + return callback(occurrence); }); - - while(it != occurrences.end()) { - auto o = safe_cast(it->occurrence()); - if(o->range.contains(offset)) { - if(!callback(*o)) { - break; - } - - it++; - continue; + } else if(self.buffer) { + auto root = kota::codec::fbs::table_view::from_bytes(buffer_bytes(*self.buffer)); + auto occurrences = root[&Impl::occurrences]; + + // Entries are sorted by Occurrence's (begin, end, target) ordering; + // mirror the in-memory lower_bound on range.end. + std::size_t lo = 0; + std::size_t hi = occurrences.size(); + while(lo < hi) { + auto mid = lo + (hi - lo) / 2; + if(occurrences.at(mid).get<0>().range.end < offset) { + lo = mid + 1; + } else { + hi = mid; } + } - break; + for(std::size_t i = lo; i < occurrences.size(); ++i) { + auto occurrence = occurrences.at(i).get<0>(); + if(!occurrence.range.contains(offset)) { + break; + } + if(!callback(occurrence)) { + break; + } } } } @@ -650,18 +501,23 @@ void MergedIndex::lookup(this const Self& self, } } } else if(self.buffer) { - auto index = fbs::GetRoot(self.buffer->getBufferStart()); - auto& entries = *index->relations(); - - auto it = std::ranges::lower_bound(entries, symbol, {}, [](auto e) { return e->symbol(); }); - if(it == entries.end() || it->symbol() != symbol) [[unlikely]] { + auto root = kota::codec::fbs::table_view::from_bytes(buffer_bytes(*self.buffer)); + auto entry = root[&Impl::relations].find(symbol); + if(!entry) [[unlikely]] { return; } - for(auto entry: *it->relations()) { - auto r = safe_cast(entry->relation()); - if(r->kind & kind) { - if(!callback(*r)) { + auto relations = entry->get<1>(); + for(std::size_t i = 0; i < relations.size(); ++i) { + auto view = relations.at(i).get<0>(); + Relation relation{ + .kind = RelationKind(static_cast(view[&Relation::kind])), + .padding = view[&Relation::padding], + .range = view[&Relation::range], + .target_symbol = view[&Relation::target_symbol], + }; + if(relation.kind & kind) { + if(!callback(relation)) { break; } } @@ -702,33 +558,37 @@ bool MergedIndex::need_update(this const Self& self) { return false; } else if(self.buffer) { - auto index = fbs::GetRoot(self.buffer->getBufferStart()); - if(index->compilation_contexts()->empty()) { + auto root = kota::codec::fbs::table_view::from_bytes(buffer_bytes(*self.buffer)); + auto contexts = root[&Impl::compilation_contexts]; + if(contexts.empty()) { return true; } - auto context = *index->compilation_contexts()->begin(); - auto* paths = index->paths(); + auto context = contexts.at(0).get<1>(); + auto paths = root[&Impl::paths]; llvm::DenseMap hashes; - if(context->dep_hashes()) { - for(auto dep: *context->dep_hashes()) { - hashes.try_emplace(dep->path_id(), dep->content_hash()); - } + auto dep_hashes = context[&CompilationContext::dep_hashes]; + for(std::size_t i = 0; i < dep_hashes.size(); ++i) { + auto dep = dep_hashes.at(i); + hashes.try_emplace(dep.path_id, dep.content_hash); } + auto build_at = context[&CompilationContext::build_at]; + auto locations = context[&CompilationContext::include_locations]; llvm::DenseSet deps; - for(auto location: *context->include_locations()) { - if(!deps.insert(location->path_id()).second) { + for(std::size_t i = 0; i < locations.size(); ++i) { + auto location = locations.at(i); + if(!deps.insert(location.path_id).second) { continue; } // A dep the table does not cover cannot be validated: rebuild. - if(!paths || location->path_id() >= paths->size()) { + if(location.path_id >= paths.size()) { return true; } - auto it = hashes.find(location->path_id()); - if(dep_stale(paths->Get(location->path_id())->string_view(), - context->build_at(), + auto it = hashes.find(location.path_id); + if(dep_stale(llvm::StringRef(paths.at(location.path_id)), + build_at, it != hashes.end() ? std::optional(it->second) : std::nullopt)) { return true; } @@ -759,13 +619,11 @@ bool MergedIndex::has_contribution(this const Self& self, llvm::StringRef contex } if(self.buffer) { - auto root = fbs::GetRoot(self.buffer->getBufferStart()); - if(!root->paths()) { - return false; - } + auto root = kota::codec::fbs::table_view::from_bytes(buffer_bytes(*self.buffer)); + auto paths = root[&Impl::paths]; std::optional local; - for(std::uint32_t i = 0; i < root->paths()->size(); ++i) { - if(llvm::StringRef(root->paths()->Get(i)->string_view()) == context_path) { + for(std::uint32_t i = 0; i < paths.size(); ++i) { + if(llvm::StringRef(paths.at(i)) == context_path) { local = i; break; } @@ -773,16 +631,8 @@ bool MergedIndex::has_contribution(this const Self& self, llvm::StringRef contex if(!local) { return false; } - for(auto entry: *root->header_contexts()) { - if(entry->path_id() == *local) { - return true; - } - } - for(auto entry: *root->compilation_contexts()) { - if(entry->path_id() == *local) { - return true; - } - } + return root[&Impl::header_contexts].contains(*local) || + root[&Impl::compilation_contexts].contains(*local); } return false; @@ -830,18 +680,12 @@ bool MergedIndex::find_symbol(this const Self& self, return true; } } else if(self.buffer) { - auto root = fbs::GetRoot(self.buffer->getBufferStart()); - if(root->symbols()) { - for(auto entry: *root->symbols()) { - if(entry->symbol_id() == hash) { - if(auto* s = entry->symbol()) { - if(s->name()) - name = s->name()->str(); - kind = SymbolKind(static_cast(s->kind())); - } - return true; - } - } + auto root = kota::codec::fbs::table_view::from_bytes(buffer_bytes(*self.buffer)); + if(auto entry = root[&Impl::symbols].find(hash)) { + auto symbol = entry->get<1>(); + name = std::string(symbol[&Symbol::name]); + kind = SymbolKind(static_cast(symbol[&Symbol::kind])); + return true; } } return false; @@ -954,10 +798,9 @@ llvm::StringRef MergedIndex::content(this const Self& self) { if(self.impl) { return self.impl->content; } else if(self.buffer) { - auto root = fbs::GetRoot(self.buffer->getBufferStart()); - if(root->content()) { - return root->content()->string_view(); - } + auto root = kota::codec::fbs::table_view::from_bytes(buffer_bytes(*self.buffer)); + auto content = root[&Impl::content]; + return llvm::StringRef(content.data(), content.size()); } return {}; } @@ -966,9 +809,10 @@ std::span MergedIndex::line_starts(this const Self& self) { if(self.impl) { return self.impl->line_starts; } else if(self.buffer) { - auto root = fbs::GetRoot(self.buffer->getBufferStart()); - if(root->line_starts() && root->line_starts()->size() > 0) { - return {root->line_starts()->data(), root->line_starts()->size()}; + auto root = kota::codec::fbs::table_view::from_bytes(buffer_bytes(*self.buffer)); + auto line_starts = root[&Impl::line_starts]; + if(auto* vector = line_starts.raw(); vector != nullptr && vector->size() > 0) { + return {vector->data(), vector->size()}; } } return {}; diff --git a/src/index/merged_index.h b/src/index/merged_index.h index 48553778b8..9fe1ba6071 100644 --- a/src/index/merged_index.h +++ b/src/index/merged_index.h @@ -49,8 +49,9 @@ class MergedIndex { /// Load merged index from disk static MergedIndex load(llvm::StringRef path); - /// Serialize it to binary format. - void serialize(this const Self& self, llvm::raw_ostream& out); + /// Serialize it to binary format. Compacts removed-masked state in place + /// first, so the written bytes and the runtime object stay equivalent. + void serialize(this Self& self, llvm::raw_ostream& out); /// Lookup the occurrence in corresponding offset. void lookup(this const Self& self, diff --git a/src/index/project_index.cpp b/src/index/project_index.cpp index 2857af3b3f..a06c98d78a 100644 --- a/src/index/project_index.cpp +++ b/src/index/project_index.cpp @@ -37,90 +37,64 @@ void ProjectIndex::serialize(this const ProjectIndex& self, llvm::raw_ostream& os, const clice::PathPool& pool, llvm::ArrayRef shards) { - fbs::FlatBufferBuilder builder(1024); + // The persisted twin: same type, but bitmaps index the embedded compact + // path table instead of the runtime pool. + ProjectIndex persisted; // Compact path table: only ids the blob actually references are written, // in first-seen order. This is where garbage paths get collected — a // path the pool accumulated but nothing references never reaches disk. llvm::DenseMap local_ids; - std::vector table; auto to_local = [&](std::uint32_t pool_id) -> std::uint32_t { - auto [it, inserted] = local_ids.try_emplace(pool_id, table.size()); + auto [it, inserted] = local_ids.try_emplace(pool_id, persisted.paths.size()); if(inserted) { - table.push_back(pool.resolve(pool_id)); + persisted.paths.emplace_back(pool.resolve(pool_id)); } return it->second; }; - llvm::SmallVector buffer; llvm::SmallVector remapped; - - auto symbols = transform(self.symbols, [&](auto&& value) { - auto& [symbol_id, symbol] = value; - + for(auto& [symbol_id, symbol]: self.symbols) { remapped.clear(); for(auto ref: symbol.reference_files) { remapped.push_back(to_local(ref)); } - roaring::Roaring local_refs(remapped.size(), remapped.data()); - - buffer.clear(); - buffer.resize_for_overwrite(local_refs.getSizeInBytes(false)); - local_refs.write(buffer.data(), false); - - return binary::CreateSymbolEntry(builder, - symbol_id, - binary::CreateSymbol(builder, - CreateString(builder, symbol.name), - symbol.kind.value(), - CreateVector(builder, buffer), - static_cast(symbol.scope))); - }); - - llvm::SmallVector shard_locals; - shard_locals.reserve(shards.size()); - for(auto shard: shards) { - shard_locals.push_back(to_local(shard)); - } - auto paths = - transform(table, [&](llvm::StringRef path) { return CreateString(builder, path); }); + auto& persisted_symbol = persisted.symbols[symbol_id]; + persisted_symbol.name = symbol.name; + persisted_symbol.kind = symbol.kind; + persisted_symbol.scope = symbol.scope; + persisted_symbol.reference_files = roaring::Roaring(remapped.size(), remapped.data()); + } - auto project_index = - binary::CreateProjectIndex(builder, - CreateVector(builder, paths), - CreateVector(builder, symbols), - builder.CreateVector(shard_locals.data(), shard_locals.size()), - index_format_version); + persisted.shards.reserve(shards.size()); + for(auto shard: shards) { + persisted.shards.push_back(to_local(shard)); + } - builder.Finish(project_index); - os.write(safe_cast(builder.GetBufferPointer()), builder.GetSize()); + write_flatbuffer(os, persisted); } std::optional ProjectIndex::from(const void* data, std::size_t size, clice::PathPool& pool, llvm::SmallVectorImpl& shards) { - fbs::Verifier verifier(static_cast(data), size); - if(!verifier.VerifyBuffer()) { + ProjectIndex loaded; + if(!read_flatbuffer(data, size, loaded)) { return std::nullopt; } - - auto root = fbs::GetRoot(data); - if(root->format_version() != index_format_version) { + if(loaded.format_version != index_format_version) { return std::nullopt; } - ProjectIndex loaded; - - // Intern the blob's compact path table into the running pool; every id - // in the blob is an index into it. + // Rebase the decoded blob onto the running pool: intern its compact path + // table, rewrite every bitmap from blob-local ids to pool ids in place, + // and hand the shard manifest out. Afterwards the object is in runtime + // form, so the persisted-only fields are cleared. llvm::SmallVector pool_ids; - if(root->paths()) { - pool_ids.reserve(root->paths()->size()); - for(auto path: *root->paths()) { - pool_ids.push_back(pool.intern(path->string_view())); - } + pool_ids.reserve(loaded.paths.size()); + for(auto& path: loaded.paths) { + pool_ids.push_back(pool.intern(path)); } auto to_pool = [&](std::uint32_t local) -> std::optional { @@ -130,34 +104,25 @@ std::optional ProjectIndex::from(const void* data, return pool_ids[local]; }; - if(root->symbols()) { - for(auto entry: *root->symbols()) { - auto* fb_symbol = entry->symbol(); - if(!fb_symbol) { - continue; - } - auto& symbol = loaded.symbols[entry->symbol_id()]; - if(auto* name = fb_symbol->name()) { - symbol.name = name->str(); - } - symbol.kind = SymbolKind(static_cast(fb_symbol->kind())); - symbol.scope = static_cast(fb_symbol->scope()); - for(auto local: read_bitmap(fb_symbol->refs())) { - if(auto id = to_pool(local)) { - symbol.reference_files.add(*id); - } + for(auto& [symbol_id, symbol]: loaded.symbols) { + roaring::Roaring global_refs; + for(auto local: symbol.reference_files) { + if(auto id = to_pool(local)) { + global_refs.add(*id); } } + symbol.reference_files = std::move(global_refs); } - if(root->shards()) { - for(auto local: *root->shards()) { - if(auto id = to_pool(local)) { - shards.push_back(*id); - } + for(auto local: loaded.shards) { + if(auto id = to_pool(local)) { + shards.push_back(*id); } } + loaded.paths.clear(); + loaded.shards.clear(); + return loaded; } diff --git a/src/index/project_index.h b/src/index/project_index.h index 50f0034a83..2a0477a6a3 100644 --- a/src/index/project_index.h +++ b/src/index/project_index.h @@ -2,7 +2,10 @@ #include #include +#include +#include +#include "index/shared.h" #include "index/tu_index.h" #include "support/path_pool.h" @@ -20,9 +23,26 @@ namespace clice::index { /// self-contained path table (which is also the garbage collection: paths no /// longer referenced by any symbol or shard are simply not written), and /// loading interns the table back into the running pool. +/// +/// The struct is its own serialization root: format_version / paths / shards +/// belong to the on-disk form, whose bitmaps index the embedded path table. +/// On the live instance they stay empty and bitmaps hold pool ids. struct ProjectIndex { + /// On-disk schema version, serialized first so loaders can reject + /// incompatible blobs before touching anything else. + std::uint32_t format_version = index_format_version; + + /// Compact self-contained path table the persisted bitmaps index into. + /// Empty at runtime. + std::vector paths; + SymbolTable symbols; + /// Blob-local ids of the files owning a MergedIndex shard, persisted so + /// the loader knows what to fetch. Empty at runtime — the live manifest + /// lives in workspace.merged_indices. + std::vector shards; + /// Merge a TU's external symbols, interning the TU's paths into `pool`. /// Returns the TU-local id → pool id mapping for the TU's path graph. llvm::SmallVector merge(this ProjectIndex& self, diff --git a/src/index/schema.fbs b/src/index/schema.fbs deleted file mode 100644 index 2262ebad46..0000000000 --- a/src/index/schema.fbs +++ /dev/null @@ -1,195 +0,0 @@ -namespace clice.index.binary; - -struct Range { - begin : uint; - end : uint; -} - -struct Occurrence { - range : Range; - target : ulong; -} - -struct Relation { - kind : uint; - padding : uint; - range : Range; - target_symbol : ulong; -} - -table CacheEntry { -sha256: - string; -canonical_id: - uint; -} - -struct IncludeContext { - include_id : uint; - canonical_id : uint; -} - -table HeaderContextEntry { -path_id: - uint; -version: - uint; -includes: - [IncludeContext]; -} - -struct IncludeLocation { - path_id : uint; - line : uint; - include_id : uint; -} - -struct DepHash { - path_id : uint; - content_hash : ulong; -} - -table CompilationContextEntry { -path_id: - uint; -version: - uint; -canonical_id: - uint; -build_at: - ulong; -include_locations: - [IncludeLocation]; -dep_hashes: - [DepHash]; -} - -table OccurrenceEntry { -occurrence: - Occurrence; -context: - [ubyte]; -} - -table RelationEntry { -relation: - Relation; -context: - [ubyte]; -} - -table SymbolRelationsEntry { -symbol: - ulong; -relations: - [RelationEntry]; -} - -table Symbol { -name: - string; -kind: - ubyte; -refs: - [ubyte] (required); -scope: - ubyte; -} - -table SymbolEntry { -symbol_id: - ulong; -symbol: - Symbol; -} - -table MergedIndex { -max_canonical_id: - uint; - -// Shard-local path table: every path id inside this shard (dependency -// locations, dependency hashes, context keys) indexes into it. Shards are -// fully self-contained — runtime pool ids are per-session and never persist. -paths: - [string] (required); - -canonical_cache: - [CacheEntry] (required); - -header_contexts: - [HeaderContextEntry] (required); - -compilation_contexts: - [CompilationContextEntry] (required); - -occurrences: - [OccurrenceEntry] (required); - -relations: - [SymbolRelationsEntry] (required); - -removed: - [ubyte]; - -content: - string; - -line_starts: - [uint]; - -symbols: - [SymbolEntry]; - -format_version: - uint; -} - -table TUFileRelationsEntry { -symbol: - ulong; -relations: - [Relation]; -} - -table TUFileIndexEntry { -file_id: - uint; -occurrences: - [Occurrence]; -relations: - [TUFileRelationsEntry]; -} - -table TUIndex { -built_at: - ulong; -paths: - [string]; -locations: - [IncludeLocation]; -symbols: - [SymbolEntry]; -file_indices: - [TUFileIndexEntry]; -main_file_index: - TUFileIndexEntry; -} - -// The blob's path table is compact: only paths referenced by the symbol -// bitmaps or the shard list are written (garbage paths are collected at -// save time), and every id in the blob is an index into it. Runtime pool -// ids are per-session and never persist. -table ProjectIndex { -paths: - [string] (required); -symbols: - [SymbolEntry] (required); - -// Path-table indices of the files owning a MergedIndex shard blob; the -// loader fetches exactly these (keyed by path hash) and sweeps the rest. -shards: - [uint] (required); - -format_version: - uint; -} diff --git a/src/index/serialization.h b/src/index/serialization.h index bf3e34f661..5efd9140ff 100644 --- a/src/index/serialization.h +++ b/src/index/serialization.h @@ -1,85 +1,142 @@ -#include -#include -#include +#pragma once -#include "schema_generated.h" +#include +#include +#include +#include +#include +#include +#include +#include + +#include "index/path_pool.h" +#include "index/shared.h" +#include "semantic/relation_kind.h" +#include "semantic/symbol_kind.h" #include "support/bitmap.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/StringRef.h" +#include "kota/codec/fbs/fbs.h" +#include "llvm/Support/raw_ostream.h" namespace clice::index { -namespace fbs = flatbuffers; +/// Encode a reflected object and write the blob to `os`. +template +void write_flatbuffer(llvm::raw_ostream& os, const T& value) { + auto encoded = kota::codec::fbs::to_flatbuffer(value); + assert(encoded && "flatbuffer serialization failed"); + if(encoded) { + os.write(reinterpret_cast(encoded->data()), encoded->size()); + } +} -/// On-disk MergedIndex shard schema version. Bump this whenever `schema.fbs` -/// changes the shard layout so that `MergedIndex::load` silently discards any -/// shard carrying a different value — including version-less shards written by -/// older builds, which read back the field's default of 0. -constexpr inline std::uint32_t index_format_version = 1; +/// Deep-verify and decode a blob into `out`; false on any malformed input. +template +bool read_flatbuffer(const void* data, std::size_t size, T& out) { + if(data == nullptr || size == 0) { + return false; + } + std::span bytes(static_cast(data), size); + return static_cast(kota::codec::fbs::from_flatbuffer(bytes, out)); +} -namespace { +} // namespace clice::index -template -concept sequence_range = std::ranges::input_range && - !requires { typename Range::key_type; } && requires(const Range& r) { - r.data(); - r.size(); - }; +/// Value-mode codec adapters bridging clice's non-reflectable leaf types onto +/// kotatsu's visitor framework. Each specialization declares the on-wire +/// layout (wire_type) plus the two conversions, and applies wherever the type +/// appears — struct field, map key/value or sequence element. The wire_type +/// also drives the FlatBuffers vector layout and verify_flatbuffer's schema +/// walk. +namespace kota::codec { -template -using Offsets = llvm::SmallVector, 0>; - -template -const U* safe_cast(const V* v) { - static_assert(sizeof(U) == sizeof(V), "size mismatch"); - static_assert(alignof(U) == alignof(V), "alignment mismatch"); - static_assert(std::is_trivially_copyable_v && std::is_trivially_copyable_v, - "requires trivially copyable"); - /// If aliasing issues arise, prefer copying into a temporary SmallVector. - return reinterpret_cast(v); -} +template +struct serialize_visit { + using wire_type = std::uint32_t; -auto CreateString(fbs::FlatBufferBuilder& builder, llvm::StringRef string) { - return builder.CreateString(string.data(), string.size()); -} + static wire_type to_wire(clice::RelationKind kind) { + return kind.value(); + } -template -auto CreateVector(fbs::FlatBufferBuilder& builder, const Range& range) { - return builder.CreateVector(range.data(), range.size()); -} + static clice::RelationKind from_wire(wire_type value) { + return clice::RelationKind(static_cast(value)); + } +}; -auto CreateVector(fbs::FlatBufferBuilder& builder, const llvm::SmallVector& range) { - return builder.CreateVector(reinterpret_cast(range.data()), range.size()); -} +template +struct serialize_visit { + using wire_type = std::uint8_t; -template -auto CreateStructVector(fbs::FlatBufferBuilder& builder, const Range& range) { - using V = std::ranges::range_value_t; - (void)sizeof(V); - return builder.CreateVectorOfStructs(safe_cast(range.data()), range.size()); -} + static wire_type to_wire(clice::SymbolKind kind) { + return kind.value(); + } -template -auto transform(const Range& range, const Functor& functor) { - using V = std::ranges::range_value_t; - using R = std::invoke_result_t; + static clice::SymbolKind from_wire(wire_type value) { + return clice::SymbolKind(static_cast(value)); + } +}; - llvm::SmallVector result; - result.resize_for_overwrite(std::ranges::size(range)); +template +struct serialize_visit { + using wire_type = std::int64_t; - auto i = 0; - for(auto&& v: range) { - result[i] = functor(v); - i += 1; + static wire_type to_wire(std::chrono::milliseconds value) { + return value.count(); } - return result; -} -Bitmap read_bitmap(const fbs::Vector* buffer) { - return Bitmap::read(reinterpret_cast(buffer->data()), false); -} + static std::chrono::milliseconds from_wire(wire_type ticks) { + return std::chrono::milliseconds(ticks); + } +}; + +/// Roaring bitmaps travel as their non-portable serialized bytes, matching the +/// `write(..., false)` format the flatc-era schema used. An empty bitmap is +/// written as an empty byte vector. +template +struct serialize_visit { + using wire_type = std::vector; + + static wire_type to_wire(const roaring::Roaring& bitmap) { + std::vector bytes; + if(!bitmap.isEmpty()) { + bytes.resize(bitmap.getSizeInBytes(false)); + bitmap.write(reinterpret_cast(bytes.data()), false); + } + return bytes; + } -} // namespace + static roaring::Roaring from_wire(const wire_type& bytes) { + if(bytes.empty()) { + return roaring::Roaring(); + } + return roaring::Roaring::read(reinterpret_cast(bytes.data()), false); + } +}; + +/// A PathPool travels as its path table (a vector of strings, in id order); +/// decoding re-interns every entry so the cache and ids are rebuilt. +template +struct serialize_visit { + using wire_type = std::vector; + + static std::vector to_wire(const clice::index::PathPool& pool) { + std::vector paths; + paths.reserve(pool.paths.size()); + for(auto path: pool.paths) { + paths.emplace_back(path.data(), path.size()); + } + return paths; + } -} // namespace clice::index + static clice::index::PathPool from_wire(const wire_type& paths) { + clice::index::PathPool pool; + for(const auto& path: paths) { + if(!path.empty()) { + pool.path_id(path); + } + } + return pool; + } +}; + +} // namespace kota::codec diff --git a/src/index/shared.h b/src/index/shared.h index a943671b99..979d72dfe1 100644 --- a/src/index/shared.h +++ b/src/index/shared.h @@ -1,5 +1,7 @@ #pragma once +#include + #include "llvm/ADT/DenseMap.h" #include "clang/Basic/SourceLocation.h" @@ -14,4 +16,10 @@ namespace clice::index { template using Shared = llvm::DenseMap; -} +/// On-disk index blob schema version. The wire layout is derived from the +/// reflected in-memory types, so bump this whenever their serialized fields +/// change; loaders silently discard blobs carrying a different value (flatc +/// era blobs are already rejected by the kotatsu buffer identifier). +constexpr inline std::uint32_t index_format_version = 2; + +} // namespace clice::index diff --git a/src/index/tu_index.cpp b/src/index/tu_index.cpp index b0943c6ed2..8b68650727 100644 --- a/src/index/tu_index.cpp +++ b/src/index/tu_index.cpp @@ -1,6 +1,8 @@ #include "index/tu_index.h" #include +#include +#include #include #include "index/serialization.h" @@ -32,6 +34,16 @@ class Builder : public SemanticVisitor { result.graph = IncludeGraph::from(unit); } + /// Per-file index for `fid`: the interested file accumulates into + /// main_file_index, every other file into its path-id slot (several + /// FileIDs of one path merge into the same entry). + FileIndex& index_of(clang::FileID fid) { + if(fid == unit.interested_file()) { + return result.main_file_index; + } + return result.path_file_indices[result.graph.path_id(fid)]; + } + void handleDeclOccurrence(const clang::NamedDecl* decl, RelationKind kind, clang::SourceLocation location) { @@ -52,7 +64,7 @@ class Builder : public SemanticVisitor { } auto [fid, range] = unit.decompose_range(location); - auto& index = result.file_indices[fid]; + auto& index = index_of(fid); auto symbol_id = unit.getSymbolID(decl); auto [it, success] = result.symbols.try_emplace(symbol_id.hash); @@ -74,7 +86,7 @@ class Builder : public SemanticVisitor { } auto [fid, range] = unit.decompose_range(location); - auto& index = result.file_indices[fid]; + auto& index = index_of(fid); auto symbol_id = unit.getSymbolID(def); index.occurrences.emplace_back(range, symbol_id.hash); @@ -120,7 +132,7 @@ class Builder : public SemanticVisitor { std::unreachable(); } - auto& index = result.file_indices[fid]; + auto& index = index_of(fid); auto symbol_id = unit.getSymbolID(ast::normalize(decl)); index.relations[symbol_id.hash].emplace_back(relation); } @@ -141,7 +153,7 @@ class Builder : public SemanticVisitor { usr += name; auto hash = llvm::xxh3_64bits(usr); - auto& index = result.file_indices[fid]; + auto& index = index_of(fid); index.occurrences.emplace_back(range, hash); Relation relation{ .kind = kind, @@ -260,43 +272,24 @@ class Builder : public SemanticVisitor { index_modules(); - for(auto& [fid, index]: result.file_indices) { + auto finalize = [&](std::uint32_t path_id, FileIndex& index) { for(auto& [symbol_id, relations]: index.relations) { - std::ranges::sort(relations, [](const Relation& lhs, const Relation& rhs) { - return std::tuple(lhs.kind.value(), - lhs.range.begin, - lhs.range.end, - lhs.target_symbol) < std::tuple(rhs.kind.value(), - rhs.range.begin, - rhs.range.end, - rhs.target_symbol); - }); - auto range = - std::ranges::unique(relations, [](const Relation& lhs, const Relation& rhs) { - return lhs.kind == rhs.kind && lhs.range == rhs.range && - lhs.target_symbol == rhs.target_symbol; - }); - relations.erase(range.begin(), range.end()); - result.symbols[symbol_id].reference_files.add(result.graph.path_id(fid)); + std::ranges::sort(relations); + auto dup = std::ranges::unique(relations); + relations.erase(dup.begin(), dup.end()); + result.symbols[symbol_id].reference_files.add(path_id); } - std::ranges::sort(index.occurrences, [](const Occurrence& lhs, const Occurrence& rhs) { - return std::tuple(lhs.range.begin, lhs.range.end, lhs.target) < - std::tuple(rhs.range.begin, rhs.range.end, rhs.target); - }); - auto range = - std::ranges::unique(index.occurrences, - [](const Occurrence& lhs, const Occurrence& rhs) { - return lhs.range == rhs.range && lhs.target == rhs.target; - }); - index.occurrences.erase(range.begin(), range.end()); - - if(fid == unit.interested_file()) { - result.main_file_index = std::move(index); - } - } + std::ranges::sort(index.occurrences); + auto dup = std::ranges::unique(index.occurrences); + index.occurrences.erase(dup.begin(), dup.end()); + }; - result.file_indices.erase(unit.interested_file()); + for(auto& [path_id, index]: result.path_file_indices) { + finalize(path_id, index); + } + /// Main file is the last path in graph.paths (convention from IncludeGraph). + finalize(static_cast(result.graph.paths.size() - 1), result.main_file_index); } private: @@ -305,8 +298,9 @@ class Builder : public SemanticVisitor { } // namespace -void FileIndex::lookup(std::uint32_t offset, - llvm::function_ref callback) const { +void lookup_occurrences(std::span occurrences, + std::uint32_t offset, + llvm::function_ref callback) { auto it = std::ranges::lower_bound(occurrences, offset, {}, [](const Occurrence& o) { return o.range.end; }); @@ -317,6 +311,11 @@ void FileIndex::lookup(std::uint32_t offset, } } +void FileIndex::lookup(std::uint32_t offset, + llvm::function_ref callback) const { + lookup_occurrences(occurrences, offset, callback); +} + void FileIndex::lookup(SymbolHash symbol, RelationKind kind, llvm::function_ref callback) const { @@ -371,120 +370,14 @@ TUIndex TUIndex::build(CompilationUnitRef unit, bool interested_only) { } void TUIndex::serialize(llvm::raw_ostream& os) const { - fbs::FlatBufferBuilder builder(4096); - - llvm::SmallVector buffer; - - auto paths = - transform(graph.paths, [&](const std::string& p) { return builder.CreateString(p); }); - - auto syms = transform(symbols, [&](auto&& value) { - auto& [symbol_id, symbol] = value; - buffer.clear(); - buffer.resize_for_overwrite(symbol.reference_files.getSizeInBytes(false)); - symbol.reference_files.write(buffer.data(), false); - return binary::CreateSymbolEntry(builder, - symbol_id, - binary::CreateSymbol(builder, - CreateString(builder, symbol.name), - symbol.kind.value(), - CreateVector(builder, buffer), - static_cast(symbol.scope))); - }); - - /// Serialize a single FileIndex into a TUFileIndexEntry. - auto serialize_file_index = [&](std::uint32_t fid, const FileIndex& index) { - auto occs = CreateStructVector(builder, index.occurrences); - auto rels = transform(index.relations, [&](auto&& value) { - auto& [symbol_id, relations] = value; - return binary::CreateTUFileRelationsEntry( - builder, - symbol_id, - CreateStructVector(builder, relations)); - }); - return binary::CreateTUFileIndexEntry(builder, fid, occs, CreateVector(builder, rels)); - }; - - /// Convert FileID-keyed file_indices to path_id-keyed entries. - llvm::SmallVector> file_idx_vec; - for(auto& [fid, index]: file_indices) { - auto pid = graph.path_id(fid); - file_idx_vec.push_back(serialize_file_index(pid, index)); - } - - /// Main file is the last path in graph.paths (convention from IncludeGraph). - auto main_idx = - serialize_file_index(static_cast(graph.paths.size() - 1), main_file_index); - - auto tu_index = - binary::CreateTUIndex(builder, - static_cast(built_at.count()), - CreateVector(builder, paths), - CreateStructVector(builder, graph.locations), - CreateVector(builder, syms), - builder.CreateVector(file_idx_vec.data(), file_idx_vec.size()), - main_idx); - - builder.Finish(tu_index); - os.write(safe_cast(builder.GetBufferPointer()), builder.GetSize()); + write_flatbuffer(os, *this); } -TUIndex TUIndex::from(const void* data) { - auto root = fbs::GetRoot(data); - +TUIndex TUIndex::from(const void* data, std::size_t size) { TUIndex index; - index.built_at = std::chrono::milliseconds(root->built_at()); - - for(auto p: *root->paths()) { - index.graph.paths.emplace_back(p->str()); - } - - for(auto loc: *root->locations()) { - index.graph.locations.emplace_back(*safe_cast(loc)); - } - - for(auto entry: *root->symbols()) { - auto& symbol = index.symbols[entry->symbol_id()]; - symbol.name = entry->symbol()->name()->str(); - symbol.kind = SymbolKind(static_cast(entry->symbol()->kind())); - symbol.scope = static_cast(entry->symbol()->scope()); - symbol.reference_files = read_bitmap(entry->symbol()->refs()); - } - - /// Helper to deserialize a TUFileIndexEntry into a FileIndex. - auto deserialize_file_index = [](const binary::TUFileIndexEntry* entry) -> FileIndex { - FileIndex fi; - if(entry->occurrences()) { - fi.occurrences.reserve(entry->occurrences()->size()); - for(auto o: *entry->occurrences()) { - fi.occurrences.emplace_back(*safe_cast(o)); - } - } - if(entry->relations()) { - for(auto rel_entry: *entry->relations()) { - auto& rels = fi.relations[rel_entry->symbol()]; - if(rel_entry->relations()) { - rels.reserve(rel_entry->relations()->size()); - for(auto r: *rel_entry->relations()) { - rels.emplace_back(*safe_cast(r)); - } - } - } - } - return fi; - }; - - /// Populate path_file_indices keyed by path_id (no clang::FileID needed). - if(root->file_indices()) { - for(auto entry: *root->file_indices()) { - index.path_file_indices[entry->file_id()] = deserialize_file_index(entry); - } - } - - if(root->main_file_index()) { - index.main_file_index = deserialize_file_index(root->main_file_index()); + if(!read_flatbuffer(data, size, index)) { + return TUIndex(); } - return index; } diff --git a/src/index/tu_index.h b/src/index/tu_index.h index d5e8bd899d..a11fbe3be1 100644 --- a/src/index/tu_index.h +++ b/src/index/tu_index.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include @@ -50,6 +51,9 @@ struct Relation { constexpr auto definition_range() { return std::bit_cast(target_symbol); } + + /// Ordering defines the serialized wire order of relation maps. + friend constexpr auto operator<=>(const Relation&, const Relation&) = default; }; struct Occurrence { @@ -60,8 +64,19 @@ struct Occurrence { SymbolHash target; friend bool operator==(const Occurrence&, const Occurrence&) = default; + + /// (begin, end, target) ordering — the serialized wire order of occurrence + /// maps, which the lazy buffer lookup binary-searches. + friend constexpr auto operator<=>(const Occurrence&, const Occurrence&) = default; }; +/// Visit every occurrence whose range contains `offset` in a sequence sorted +/// by Occurrence's (begin, end, target) ordering; stops early when the +/// callback returns false. +void lookup_occurrences(std::span occurrences, + std::uint32_t offset, + llvm::function_ref callback); + struct FileIndex { llvm::DenseMap> relations; @@ -100,10 +115,8 @@ struct TUIndex { SymbolTable symbols; - llvm::DenseMap file_indices; - - /// File indices keyed by path_id, populated by from() for deserialized data. - /// When built from AST, this is empty and file_indices (keyed by FileID) is used. + /// Per-file indexes keyed by path id (files of the same path merge into + /// one entry); the main file lives in main_file_index instead. llvm::DenseMap path_file_indices; FileIndex main_file_index; @@ -112,7 +125,7 @@ struct TUIndex { void serialize(llvm::raw_ostream& os) const; - static TUIndex from(const void* data); + static TUIndex from(const void* data, std::size_t size); }; } // namespace clice::index diff --git a/src/semantic/relation_kind.h b/src/semantic/relation_kind.h index 9916a0ca47..3585a0a98f 100644 --- a/src/semantic/relation_kind.h +++ b/src/semantic/relation_kind.h @@ -71,6 +71,12 @@ constexpr bool operator==(RelationKind lhs, RelationKind rhs) { return lhs.value() == rhs.value(); } +/// Ordering by raw kind value; lets Relation default its own operator<=> so +/// serialized relation maps have a deterministic, binary-searchable order. +constexpr auto operator<=>(RelationKind lhs, RelationKind rhs) { + return lhs.value() <=> rhs.value(); +} + constexpr bool operator&(RelationKind lhs, RelationKind rhs) { return lhs.value() == rhs.value(); } diff --git a/src/server/compiler/compiler.cpp b/src/server/compiler/compiler.cpp index 1143a97cbb..c5c6b2187b 100644 --- a/src/server/compiler/compiler.cpp +++ b/src/server/compiler/compiler.cpp @@ -723,7 +723,8 @@ kota::task<> Compiler::run_compile(std::shared_ptr session) { record_deps(*session, result.value().deps); if(!result.value().tu_index_data.empty()) { - auto tu_index = index::TUIndex::from(result.value().tu_index_data.data()); + auto tu_index = index::TUIndex::from(result.value().tu_index_data.data(), + result.value().tu_index_data.size()); session->file_index = std::move(tu_index.main_file_index); session->symbols = std::move(tu_index.symbols); } diff --git a/src/server/index/background_indexer.cpp b/src/server/index/background_indexer.cpp index c65a0b6d48..22e9bd84ac 100644 --- a/src/server/index/background_indexer.cpp +++ b/src/server/index/background_indexer.cpp @@ -27,7 +27,7 @@ namespace clice { void BackgroundIndexer::merge(const void* tu_index_data, std::size_t size) { - auto tu_index = index::TUIndex::from(tu_index_data); + auto tu_index = index::TUIndex::from(tu_index_data, size); if(tu_index.graph.paths.empty()) { LOG_WARN("Ignoring TUIndex with empty path graph"); return; @@ -84,13 +84,7 @@ void BackgroundIndexer::merge(const void* tu_index_data, std::size_t size) { shard.merge_symbols(collect_local_symbols(file_idx)); touched.insert(global_path_id); } else { - std::optional include_id; - for(std::uint32_t i = 0; i < tu_index.graph.locations.size(); ++i) { - if(tu_index.graph.locations[i].path_id == tu_path_id) { - include_id = i; - break; - } - } + auto include_id = tu_index.graph.first_include_of(tu_path_id); if(!include_id) { LOG_WARN("Skip merge for path {}: include location not found", global_path_id); return; diff --git a/src/syntax/token.h b/src/syntax/token.h index ad844fadc7..fba41674f0 100644 --- a/src/syntax/token.h +++ b/src/syntax/token.h @@ -46,6 +46,8 @@ struct LocalSourceRange { constexpr bool operator==(const LocalSourceRange& other) const = default; + constexpr auto operator<=>(const LocalSourceRange& other) const = default; + constexpr std::uint32_t length() const { return end - begin; } diff --git a/tests/unit/index/index_query_tests.cpp b/tests/unit/index/index_query_tests.cpp index 0476be4096..13a198d022 100644 --- a/tests/unit/index/index_query_tests.cpp +++ b/tests/unit/index/index_query_tests.cpp @@ -39,11 +39,11 @@ void build_and_merge(llvm::StringRef code, {}); // Merge header file indices. - for(auto& [fid, file_idx]: tu_index.file_indices) { - auto tu_pid = tu_index.graph.path_id(fid); + for(auto& [tu_pid, file_idx]: tu_index.path_file_indices) { + auto include_id = tu_index.graph.first_include_of(tu_pid); + ASSERT_TRUE(include_id.has_value()); auto global_pid = file_ids_map[tu_pid]; - auto include_id = tu_index.graph.include_location_id(fid); - merged_indices[global_pid].merge(main_tu_path, include_id, file_idx, {}); + merged_indices[global_pid].merge(main_tu_path, *include_id, file_idx, {}); } } @@ -341,11 +341,11 @@ TEST_CASE(CrossFileQuery) { {}); // Merge header file indices. - for(auto& [fid, file_idx]: tu_index.file_indices) { - auto tu_pid = tu_index.graph.path_id(fid); + for(auto& [tu_pid, file_idx]: tu_index.path_file_indices) { + auto include_id = tu_index.graph.first_include_of(tu_pid); + ASSERT_TRUE(include_id.has_value()); auto global_pid = file_ids_map[tu_pid]; - auto include_id = tu_index.graph.include_location_id(fid); - merged_indices[global_pid].merge(main_tu_path, include_id, file_idx, {}); + merged_indices[global_pid].merge(main_tu_path, *include_id, file_idx, {}); } // Query: from usage in main.cpp, find the symbol via merged index. diff --git a/tests/unit/index/merged_index_tests.cpp b/tests/unit/index/merged_index_tests.cpp index e3734b6029..b3405cd381 100644 --- a/tests/unit/index/merged_index_tests.cpp +++ b/tests/unit/index/merged_index_tests.cpp @@ -1,10 +1,10 @@ #include -#include "schema_generated.h" #include "test/temp_dir.h" #include "test/test.h" #include "test/tester.h" #include "index/merged_index.h" +#include "index/serialization.h" #include "llvm/Support/raw_ostream.h" @@ -24,6 +24,15 @@ void build_index(llvm::StringRef code, tu_index = index::TUIndex::build(*unit); }; +/// File index of `fid`: the main file lives in main_file_index, every other +/// file is keyed by its path id. +index::FileIndex& file_index_of(index::TUIndex& index, clang::FileID fid) { + if(fid == unit->interested_file()) { + return index.main_file_index; + } + return index.path_file_indices[index.graph.path_id(fid)]; +} + void EXPECT_SELECT(llvm::StringRef pos, llvm::StringRef expect_range, llvm::StringRef file = "", @@ -32,7 +41,7 @@ void EXPECT_SELECT(llvm::StringRef pos, auto expected = range(expect_range, file); auto fid = file.empty() ? unit->interested_file() : unit->file_id(file); - auto& index = tu_index.file_indices[fid]; + auto& index = file_index_of(tu_index, fid); auto it = std::ranges::lower_bound(index.occurrences, offset, {}, [](index::Occurrence& occurrence) { @@ -58,9 +67,10 @@ TEST_CASE(Serialization) { llvm::StringMap merged_indices; auto& graph = tu_index.graph; - for(auto& [fid, index]: tu_index.file_indices) { - llvm::StringRef path = graph.paths[graph.path_id(fid)]; - merged_indices[path].merge("tu0", graph.include_location_id(fid), index, {}); + for(auto& [path_id, index]: tu_index.path_file_indices) { + auto include_id = graph.first_include_of(path_id); + ASSERT_TRUE(include_id.has_value()); + merged_indices[graph.paths[path_id]].merge("tu0", *include_id, index, {}); } for(auto& [path, merged]: merged_indices) { @@ -153,11 +163,11 @@ TEST_CASE(MultipleMergesDedup) { // Merge header indices from both TUs into same MergedIndex. index::MergedIndex merged_header; - for(auto& [fid, file_index]: tu_a.file_indices) { - merged_header.merge("tu0", tu_a.graph.include_location_id(fid), file_index, {}); + for(auto& [path_id, file_index]: tu_a.path_file_indices) { + merged_header.merge("tu0", *tu_a.graph.first_include_of(path_id), file_index, {}); } - for(auto& [fid, file_index]: tu_b.file_indices) { - merged_header.merge("tu1", tu_b.graph.include_location_id(fid), file_index, {}); + for(auto& [path_id, file_index]: tu_b.path_file_indices) { + merged_header.merge("tu1", *tu_b.graph.first_include_of(path_id), file_index, {}); } // Serialize and deserialize to verify dedup survives round-trip. @@ -253,8 +263,8 @@ TEST_CASE(RemoveHeaderContext) { // Merge header index as header context. index::MergedIndex merged_header; - for(auto& [fid, file_index]: tu_index.file_indices) { - merged_header.merge("tu0", tu_index.graph.include_location_id(fid), file_index, {}); + for(auto& [path_id, file_index]: tu_index.path_file_indices) { + merged_header.merge("tu0", *tu_index.graph.first_include_of(path_id), file_index, {}); } // Remove should not crash. @@ -279,7 +289,7 @@ TEST_CASE(RemergeReplacesContribution) { tu_index = index::TUIndex::build(*unit); auto header_fid = unit->file_id("header.h"); - auto& header_idx = tu_index.file_indices[header_fid]; + auto& header_idx = file_index_of(tu_index, header_fid); auto include_id = tu_index.graph.include_location_id(header_fid); // The symbol defined in the header: its Definition relation exists only @@ -329,7 +339,7 @@ TEST_CASE(RemergePreservesOtherTus) { tu_index = index::TUIndex::build(*unit); auto header_fid = unit->file_id("header.h"); - auto& header_idx = tu_index.file_indices[header_fid]; + auto& header_idx = file_index_of(tu_index, header_fid); auto include_id = tu_index.graph.include_location_id(header_fid); index::SymbolHash defined{}; @@ -395,7 +405,7 @@ TEST_CASE(HasContributionTracking) { tu_index = index::TUIndex::build(*unit); auto header_fid = unit->file_id("header.h"); - auto& header_idx = tu_index.file_indices[header_fid]; + auto& header_idx = file_index_of(tu_index, header_fid); auto include_id = tu_index.graph.include_location_id(header_fid); index::MergedIndex merged; @@ -637,39 +647,36 @@ TEST_CASE(OldShardDiscarded) { ASSERT_TRUE(index::MergedIndex::load(path).content() == "valid-shard"); } - // A version-less (format_version=0) shard from an older build is silently - // discarded — load returns an empty index, as if nothing were on disk. + // A shard carrying a stale format_version is silently discarded — load + // returns an empty index, as if nothing were on disk. The stale shard is + // forged from a struct mirroring the shard layout's leading fields. { - namespace binary = clice::index::binary; - flatbuffers::FlatBufferBuilder builder; - auto content = builder.CreateString("stale-shard"); - auto paths = builder.CreateVector>({}); - auto cache = builder.CreateVector>({}); - auto headers = builder.CreateVector>({}); - auto compilations = - builder.CreateVector>({}); - auto occurrences = builder.CreateVector>({}); - auto relations = - builder.CreateVector>({}); - auto root = binary::CreateMergedIndex(builder, - 0, - paths, - cache, - headers, - compilations, - occurrences, - relations, - 0, - content, - 0, - 0, - /*format_version=*/0); - builder.Finish(root); + struct StaleShard { + std::uint32_t format_version = 0; + std::vector paths; + std::string content = "stale-shard"; + }; + + auto encoded = kota::codec::fbs::to_flatbuffer(StaleShard{}); + ASSERT_TRUE(encoded.has_value()); auto path = dir.path("stale.idx"); std::error_code ec; llvm::raw_fd_ostream os(path, ec); - os.write(reinterpret_cast(builder.GetBufferPointer()), builder.GetSize()); + os.write(reinterpret_cast(encoded->data()), encoded->size()); + os.flush(); + + auto loaded = index::MergedIndex::load(path); + ASSERT_TRUE(loaded.content().empty()); + ASSERT_TRUE(loaded.need_update()); + } + + // Garbage bytes are rejected by the deep verifier. + { + auto path = dir.path("garbage.idx"); + std::error_code ec; + llvm::raw_fd_ostream os(path, ec); + os << "EVTOnot-a-flatbuffer-at-all-just-some-garbage-bytes"; os.flush(); auto loaded = index::MergedIndex::load(path); diff --git a/tests/unit/index/tu_index_tests.cpp b/tests/unit/index/tu_index_tests.cpp index 8c96d185c2..c54bb5637f 100644 --- a/tests/unit/index/tu_index_tests.cpp +++ b/tests/unit/index/tu_index_tests.cpp @@ -30,8 +30,9 @@ void build_index(llvm::StringRef code, auto select(llvm::StringRef pos, llvm::StringRef file = "") -> std::vector { auto offset = point(pos, file); auto fid = file.empty() ? unit->interested_file() : unit->file_id(file); - auto& index = - fid == unit->interested_file() ? tu_index.main_file_index : tu_index.file_indices[fid]; + auto& index = fid == unit->interested_file() + ? tu_index.main_file_index + : tu_index.path_file_indices[tu_index.graph.path_id(fid)]; auto it = std::ranges::lower_bound(index.occurrences, offset, {}, [](index::Occurrence& occurrence) { @@ -80,8 +81,9 @@ void GO_TO_DEFINITION(llvm::StringRef pos, /// dump(expected)); auto fid = file.empty() ? unit->interested_file() : unit->file_id(file); - auto& index = - fid == unit->interested_file() ? tu_index.main_file_index : tu_index.file_indices[fid]; + auto& index = fid == unit->interested_file() + ? tu_index.main_file_index + : tu_index.path_file_indices[tu_index.graph.path_id(fid)]; auto it = index.relations.find(occurrences.front().target); ASSERT_TRUE(it != index.relations.end()); @@ -301,7 +303,7 @@ TEST_CASE(BaseAndDerived) { }; collect_kinds(tu_index.main_file_index); - for(auto& [fid, idx]: tu_index.file_indices) { + for(auto& [pid, idx]: tu_index.path_file_indices) { collect_kinds(idx); } @@ -395,7 +397,7 @@ TEST_CASE(OverrideRelation) { }; check_relations(tu_index.main_file_index); - for(auto& [fid, idx]: tu_index.file_indices) { + for(auto& [pid, idx]: tu_index.path_file_indices) { check_relations(idx); } @@ -450,7 +452,7 @@ TEST_CASE(CrossFileHeaderIndex) { tu_index = index::TUIndex::build(*unit); // The header should have its own FileIndex (separate from main). - ASSERT_TRUE(tu_index.file_indices.size() >= 1U); + ASSERT_TRUE(tu_index.path_file_indices.size() >= 1U); // The main file should have a reference to helper. auto& main_index = tu_index.main_file_index; @@ -471,7 +473,7 @@ TEST_CASE(CrossFileHeaderIndex) { // The helper's declaration should be in the header FileIndex. bool found_in_header = false; - for(auto& [fid, file_index]: tu_index.file_indices) { + for(auto& [pid, file_index]: tu_index.path_file_indices) { for(auto& [sym, rels]: file_index.relations) { if(sym == helper_hash) { found_in_header = true;