Skip to content

Commit f748684

Browse files
lucylqmeta-codesync[bot]
lucylq
authored and
meta-codesync[bot]
committed
Introduce public MergedDataMap
Summary: Add public merged data map. Module can use this to resolve multiple named data maps. Creating as a sep dependency rather than inside module/ so it can be used independently of module. (think there may be some other internal usages soon) Add support for BUCK and CMake. On CMake side, have to add the dep for everyone who uses module .. Reviewed By: swolchok Differential Revision: D83527299
1 parent 0bfb61e commit f748684

31 files changed

+611
-1
lines changed

‎.ci/scripts/build-qnn-sdk.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ set_up_aot() {
3838
-DEXECUTORCH_BUILD_EXTENSION_EXTENSION_LLM=ON \
3939
-DEXECUTORCH_BUILD_EXTENSION_EXTENSION_LLM_RUNNER=ON \
4040
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
41+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
4142
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
4243
-DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
4344
-DPYTHON_EXECUTABLE=python3

‎.ci/scripts/test_llama_torchao_lowbit.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ cmake -DPYTHON_EXECUTABLE=python \
3131
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
3232
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
3333
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
34+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
3435
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
3536
-DEXECUTORCH_BUILD_XNNPACK=OFF \
3637
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \

‎.ci/scripts/test_torchao_huggingface_checkpoints.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ if [[ "$TEST_WITH_RUNNER" -eq 1 ]]; then
129129
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
130130
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
131131
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
132+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
132133
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
133134
-DEXECUTORCH_BUILD_XNNPACK=ON \
134135
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \

‎.ci/scripts/test_yolo12.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,8 @@ cmake_install_executorch_libraries() {
119119
-DEXECUTORCH_BUILD_XNNPACK="$XNNPACK" \
120120
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
121121
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
122+
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
123+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
122124
-DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
123125
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
124126
-B"${build_dir}"
@@ -131,6 +133,8 @@ cmake_install_executorch_libraries() {
131133
-DEXECUTORCH_BUILD_XNNPACK="$XNNPACK" \
132134
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
133135
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
136+
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
137+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
134138
-DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
135139
-DEXECUTORCH_ENABLE_LOGGING=ON \
136140
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \

‎.github/workflows/trunk.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -852,6 +852,7 @@ jobs:
852852
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
853853
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
854854
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
855+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
855856
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
856857
-DEXECUTORCH_BUILD_XNNPACK=ON \
857858
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \

‎CMakeLists.txt‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -630,6 +630,11 @@ if(EXECUTORCH_BUILD_EXTENSION_MODULE)
630630
list(APPEND _executorch_extensions extension_module_static)
631631
endif()
632632

633+
if(EXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP)
634+
add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extension/named_data_map)
635+
list(APPEND _executorch_extensions extension_named_data_map)
636+
endif()
637+
633638
if(EXECUTORCH_BUILD_EXTENSION_LLM)
634639
if(EXECUTORCH_BUILD_EXTENSION_LLM_RUNNER)
635640
set(SUPPORT_REGEX_LOOKAHEAD ON)

‎backends/mediatek/scripts/mtk_build.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ cmake -DCMAKE_INSTALL_PREFIX="${build_dir}" \
3030
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
3131
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
3232
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
33+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
3334
-DEXECUTORCH_BUILD_NEURON=ON \
3435
-B"${build_dir}"
3536

‎backends/qualcomm/scripts/build.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ if [ "$BUILD_AARCH64" = true ]; then
8686
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
8787
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
8888
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
89+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
8990
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
9091
-DEXECUTORCH_ENABLE_EVENT_TRACER=ON \
9192
-DEXECUTORCH_ENABLE_LOGGING=ON \
@@ -155,6 +156,7 @@ if [ "$BUILD_X86_64" = true ]; then
155156
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
156157
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
157158
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
159+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
158160
-DEXECUTORCH_BUILD_KERNELS_QUANTIZED=ON \
159161
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
160162
-DEXECUTORCH_ENABLE_EVENT_TRACER=ON \

‎backends/samsung/build.sh‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@ function build_x86_64() {
4545
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
4646
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
4747
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
48+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
4849
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
4950
-S ${PROJECT_DIR} \
5051
-B ${X86_64_BUILD_DIR}
@@ -77,6 +78,7 @@ function build_android() {
7778
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
7879
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
7980
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
81+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
8082
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
8183
-DEXECUTORCH_ENABLE_LOGGING=1 \
8284
-DEXECUTORCH_BUILD_DEVTOOLS=ON \

‎backends/vulkan/test/scripts/test_model.sh‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -111,6 +111,7 @@ build_core_libraries_and_devtools() {
111111
-DEXECUTORCH_BUILD_EXTENSION_DATA_LOADER=ON \
112112
-DEXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR=ON \
113113
-DEXECUTORCH_BUILD_EXTENSION_MODULE=ON \
114+
-DEXECUTORCH_BUILD_EXTENSION_NAMED_DATA_MAP=ON \
114115
-DEXECUTORCH_BUILD_EXTENSION_RUNNER_UTIL=ON \
115116
-DEXECUTORCH_BUILD_EXTENSION_TENSOR=ON \
116117
-DEXECUTORCH_BUILD_EXECUTOR_RUNNER=ON \

0 commit comments

Comments
 (0)