Skip to content

Commit 9113109

Browse files
digantdesaidigantdesai
authored
Merge branch 'main' into change-987432
2 parents a5e51e7 + efd1a06 commit 9113109

File tree

72 files changed

+580
-337
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

72 files changed

+580
-337
lines changed

‎.ci/scripts/test_model.sh‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -164,6 +164,7 @@ test_model_with_qnn() {
164164
export LD_LIBRARY_PATH=$QNN_SDK_ROOT/lib/x86_64-linux-clang/
165165
export PYTHONPATH=$EXECUTORCH_ROOT/..
166166

167+
EXTRA_FLAGS=""
167168
if [[ "${MODEL_NAME}" == "dl3" ]]; then
168169
EXPORT_SCRIPT=deeplab_v3
169170
elif [[ "${MODEL_NAME}" == "mv3" ]]; then
@@ -176,6 +177,12 @@ test_model_with_qnn() {
176177
EXPORT_SCRIPT=inception_v3
177178
elif [[ "${MODEL_NAME}" == "vit" ]]; then
178179
EXPORT_SCRIPT=torchvision_vit
180+
elif [[ "${MODEL_NAME}" == "mb" ]]; then
181+
EXPORT_SCRIPT=mobilebert_fine_tune
182+
EXTRA_FLAGS="--num_epochs 1"
183+
pip install scikit-learn
184+
elif [[ "${MODEL_NAME}" == "w2l" ]]; then
185+
EXPORT_SCRIPT=wav2letter
179186
elif [[ "${MODEL_NAME}" == "edsr" ]]; then
180187
EXPORT_SCRIPT=edsr
181188
# Additional deps for edsr
@@ -189,7 +196,7 @@ test_model_with_qnn() {
189196
# TODO(guangyang): Make QNN chipset matches the target device
190197
QNN_CHIPSET=SM8450
191198

192-
"${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --compile_only
199+
"${PYTHON_EXECUTABLE}" -m examples.qualcomm.scripts.${EXPORT_SCRIPT} -b ${CMAKE_OUTPUT_DIR} -m ${QNN_CHIPSET} --compile_only $EXTRA_FLAGS
193200
EXPORTED_MODEL=$(find "./${EXPORT_SCRIPT}" -type f -name "${MODEL_NAME}*.pte" -print -quit)
194201
}
195202

‎.github/workflows/android-perf.yml‎

Lines changed: 63 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -96,63 +96,6 @@ jobs:
9696
9797
PYTHONPATH="${PWD}" python .ci/scripts/gather_benchmark_configs.py $ARGS
9898
99-
prepare-test-specs:
100-
runs-on: linux.2xlarge
101-
needs: set-parameters
102-
strategy:
103-
matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }}
104-
fail-fast: false
105-
steps:
106-
- uses: actions/checkout@v3
107-
108-
- name: Prepare the spec
109-
id: prepare
110-
shell: bash
111-
env:
112-
BENCHMARK_CONFIG: ${{ toJSON(matrix) }}
113-
working-directory: extension/benchmark/android/benchmark
114-
run: |
115-
set -eux
116-
117-
# The model will be exported in the next step to this S3 path
118-
MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip"
119-
# We could write a script to properly use jinja here, but there is only one variable,
120-
# so let's just sed it
121-
sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2
122-
123-
BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g')
124-
# The config for this benchmark runs, we save it in the test spec so that it can be fetched
125-
# later by the upload script
126-
sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2
127-
128-
cp android-llm-device-farm-test-spec.yml.j2 android-llm-device-farm-test-spec.yml
129-
# Just print the test spec for debugging
130-
cat android-llm-device-farm-test-spec.yml
131-
132-
# Save the benchmark configs so that we can use it later in the dashboard
133-
echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json"
134-
echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT
135-
136-
- name: Upload the spec
137-
uses: seemethere/upload-artifact-s3@v5
138-
with:
139-
s3-bucket: gha-artifacts
140-
s3-prefix: |
141-
${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}
142-
retention-days: 1
143-
if-no-files-found: error
144-
path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
145-
146-
- name: Update the benchmark configs
147-
uses: seemethere/upload-artifact-s3@v5
148-
with:
149-
s3-bucket: gha-artifacts
150-
s3-prefix: |
151-
${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/
152-
retention-days: 1
153-
if-no-files-found: error
154-
path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json
155-
15699
export-models:
157100
name: export-models
158101
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main
@@ -335,6 +278,69 @@ jobs:
335278
fi
336279
echo "::endgroup::"
337280
281+
prepare-test-specs:
282+
runs-on: linux.2xlarge
283+
needs:
284+
- set-parameters
285+
- export-models
286+
strategy:
287+
matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }}
288+
fail-fast: false
289+
steps:
290+
- uses: actions/checkout@v3
291+
292+
- name: Prepare the spec
293+
id: prepare
294+
shell: bash
295+
env:
296+
BENCHMARK_CONFIG: ${{ toJSON(matrix) }}
297+
working-directory: extension/benchmark/android/benchmark
298+
run: |
299+
set -eux
300+
301+
# The model will be exported in the next step to this S3 path
302+
MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip"
303+
304+
# Check if the model artifact exists, fail this step skip generating test-spec.
305+
curl -s --head -f ${MODEL_PATH}
306+
307+
# We could write a script to properly use jinja here, but there is only one variable,
308+
# so let's just sed it
309+
sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' android-llm-device-farm-test-spec.yml.j2
310+
311+
BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g')
312+
# The config for this benchmark runs, we save it in the test spec so that it can be fetched
313+
# later by the upload script
314+
sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' android-llm-device-farm-test-spec.yml.j2
315+
316+
cp android-llm-device-farm-test-spec.yml.j2 android-llm-device-farm-test-spec.yml
317+
# Just print the test spec for debugging
318+
cat android-llm-device-farm-test-spec.yml
319+
320+
# Save the benchmark configs so that we can use it later in the dashboard
321+
echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json"
322+
echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT
323+
324+
- name: Upload the spec
325+
uses: seemethere/upload-artifact-s3@v5
326+
with:
327+
s3-bucket: gha-artifacts
328+
s3-prefix: |
329+
${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}
330+
retention-days: 1
331+
if-no-files-found: error
332+
path: extension/benchmark/android/benchmark/android-llm-device-farm-test-spec.yml
333+
334+
- name: Update the benchmark configs
335+
uses: seemethere/upload-artifact-s3@v5
336+
with:
337+
s3-bucket: gha-artifacts
338+
s3-prefix: |
339+
${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/
340+
retention-days: 1
341+
if-no-files-found: error
342+
path: extension/benchmark/android/benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json
343+
338344
build-benchmark-app:
339345
name: build-benchmark-app
340346
uses: pytorch/test-infra/.github/workflows/linux_job_v2.yml@main

‎.github/workflows/apple-perf.yml‎

Lines changed: 62 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -98,63 +98,6 @@ jobs:
9898
9999
echo "benchmark_configs is: ${{ steps.set-parameters.outputs.benchmark_configs }}"
100100
101-
prepare-test-specs:
102-
runs-on: linux.2xlarge
103-
needs: set-parameters
104-
strategy:
105-
matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }}
106-
fail-fast: false
107-
steps:
108-
- uses: actions/checkout@v3
109-
110-
- name: Prepare the spec
111-
id: prepare
112-
shell: bash
113-
env:
114-
BENCHMARK_CONFIG: ${{ toJSON(matrix) }}
115-
working-directory: extension/benchmark/apple/Benchmark
116-
run: |
117-
set -eux
118-
119-
# The model will be exported in the next step to this S3 path
120-
MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip"
121-
# We could write a script to properly use jinja here, but there is only one variable,
122-
# so let's just sed it
123-
sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2
124-
125-
BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g')
126-
# The config for this benchmark runs, we save it in the test spec so that it can be fetched
127-
# later by the upload script
128-
sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2
129-
130-
cp default-ios-device-farm-appium-test-spec.yml.j2 default-ios-device-farm-appium-test-spec.yml
131-
# Just print the test spec for debugging
132-
cat default-ios-device-farm-appium-test-spec.yml
133-
134-
# Save the benchmark configs so that we can use it later in the dashboard
135-
echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json"
136-
echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT
137-
138-
- name: Upload the spec
139-
uses: seemethere/upload-artifact-s3@v5
140-
with:
141-
s3-bucket: gha-artifacts
142-
s3-prefix: |
143-
${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}
144-
retention-days: 1
145-
if-no-files-found: error
146-
path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
147-
148-
- name: Update the benchmark configs
149-
uses: seemethere/upload-artifact-s3@v5
150-
with:
151-
s3-bucket: gha-artifacts
152-
s3-prefix: |
153-
${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/
154-
retention-days: 1
155-
if-no-files-found: error
156-
path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json
157-
158101
export-models:
159102
name: export-models
160103
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
@@ -344,6 +287,68 @@ jobs:
344287
fi
345288
echo "::endgroup::"
346289
290+
prepare-test-specs:
291+
runs-on: linux.2xlarge
292+
needs:
293+
- set-parameters
294+
- export-models
295+
strategy:
296+
matrix: ${{ fromJson(needs.set-parameters.outputs.benchmark_configs) }}
297+
fail-fast: false
298+
steps:
299+
- uses: actions/checkout@v3
300+
301+
- name: Prepare the spec
302+
id: prepare
303+
shell: bash
304+
env:
305+
BENCHMARK_CONFIG: ${{ toJSON(matrix) }}
306+
working-directory: extension/benchmark/apple/Benchmark
307+
run: |
308+
set -eux
309+
310+
# The model will be exported in the next step to this S3 path
311+
MODEL_PATH="https://gha-artifacts.s3.amazonaws.com/${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}/model.zip"
312+
# Check if the model artifact exists, fail this step skip generating test-spec.
313+
curl -s --head -f ${MODEL_PATH}
314+
# We could write a script to properly use jinja here, but there is only one variable,
315+
# so let's just sed it
316+
sed -i -e 's,{{ model_path }},'"${MODEL_PATH}"',g' default-ios-device-farm-appium-test-spec.yml.j2
317+
318+
BENCHMARK_CONFIG_ID=$(echo "${{ matrix.model }}_${{ matrix.config }}" | sed -e 's/[^A-Za-z0-9._-]/_/g')
319+
# The config for this benchmark runs, we save it in the test spec so that it can be fetched
320+
# later by the upload script
321+
sed -i -e 's,{{ benchmark_config_id }},'"${BENCHMARK_CONFIG_ID}"',g' default-ios-device-farm-appium-test-spec.yml.j2
322+
323+
cp default-ios-device-farm-appium-test-spec.yml.j2 default-ios-device-farm-appium-test-spec.yml
324+
# Just print the test spec for debugging
325+
cat default-ios-device-farm-appium-test-spec.yml
326+
327+
# Save the benchmark configs so that we can use it later in the dashboard
328+
echo "${BENCHMARK_CONFIG}" > "${BENCHMARK_CONFIG_ID}.json"
329+
echo "benchmark-config-id=${BENCHMARK_CONFIG_ID}" >> $GITHUB_OUTPUT
330+
331+
- name: Upload the spec
332+
uses: seemethere/upload-artifact-s3@v5
333+
with:
334+
s3-bucket: gha-artifacts
335+
s3-prefix: |
336+
${{ github.repository }}/${{ github.run_id }}/artifacts/${{ matrix.model }}_${{ matrix.config }}
337+
retention-days: 1
338+
if-no-files-found: error
339+
path: extension/benchmark/apple/Benchmark/default-ios-device-farm-appium-test-spec.yml
340+
341+
- name: Update the benchmark configs
342+
uses: seemethere/upload-artifact-s3@v5
343+
with:
344+
s3-bucket: gha-artifacts
345+
s3-prefix: |
346+
${{ github.repository }}/${{ github.run_id }}/artifacts/benchmark-configs/
347+
retention-days: 1
348+
if-no-files-found: error
349+
path: extension/benchmark/apple/Benchmark/${{ steps.prepare.outputs.benchmark-config-id }}.json
350+
351+
347352
build-benchmark-app:
348353
name: build-benchmark-app
349354
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main

‎.github/workflows/trunk.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,7 @@ jobs:
311311
strategy:
312312
matrix:
313313
dtype: [fp32]
314-
model: [dl3, mv3, mv2, ic4, ic3, vit]
314+
model: [dl3, mv3, mv2, ic4, ic3, vit, mb, w2l]
315315
fail-fast: false
316316
with:
317317
runner: linux.2xlarge

‎CMakeLists.txt‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -248,14 +248,15 @@ cmake_dependent_option(
248248
"NOT EXECUTORCH_BUILD_ARM_BAREMETAL" OFF
249249
)
250250

251-
if(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR)
251+
if(EXECUTORCH_BUILD_EXTENSION_TRAINING)
252252
set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON)
253+
set(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR ON)
254+
set(EXECUTORCH_BUILD_EXTENSION_MODULE ON)
255+
set(EXECUTORCH_BUILD_EXTENSION_TENSOR ON)
253256
endif()
254257

255-
if(EXECUTORCH_BUILD_EXTENSION_TRAINING)
256-
set(EXECUTORCH_BUILD_EXTENSION_TENSOR ON)
258+
if(EXECUTORCH_BUILD_EXTENSION_FLAT_TENSOR)
257259
set(EXECUTORCH_BUILD_EXTENSION_DATA_LOADER ON)
258-
set(EXECUTORCH_BUILD_EXTENSION_MODULE ON)
259260
endif()
260261

261262
if(EXECUTORCH_BUILD_EXTENSION_MODULE)

‎backends/apple/coreml/TARGETS‎

Lines changed: 8 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ runtime.python_library(
1414
"@EXECUTORCH_CLIENTS",
1515
],
1616
deps = [
17+
"fbsource//third-party/pypi/coremltools:coremltools",
1718
":executorchcoreml",
1819
"//executorch/exir/backend:backend_details",
1920
"//executorch/exir/backend:compile_spec_schema",
20-
"fbsource//third-party/pypi/coremltools:coremltools",
2121
],
2222
)
2323

@@ -30,13 +30,13 @@ runtime.python_library(
3030
"@EXECUTORCH_CLIENTS",
3131
],
3232
deps = [
33+
"fbsource//third-party/pypi/coremltools:coremltools",
3334
":backend",
3435
"//caffe2:torch",
3536
"//executorch/exir:lib",
3637
"//executorch/exir/backend:compile_spec_schema",
3738
"//executorch/exir/backend:partitioner",
3839
"//executorch/exir/backend:utils",
39-
"fbsource//third-party/pypi/coremltools:coremltools",
4040
],
4141
)
4242

@@ -64,25 +64,23 @@ runtime.cxx_python_extension(
6464
headers = glob([
6565
"runtime/inmemoryfs/**/*.hpp",
6666
]),
67+
base_module = "",
68+
compiler_flags = [
69+
"-std=c++17",
70+
],
6771
preprocessor_flags = [
6872
"-Iexecutorch/backends/apple/coreml/runtime/util",
6973
],
7074
types = [
7175
"executorchcoreml.pyi",
7276
],
73-
compiler_flags = [
74-
"-std=c++17",
75-
],
76-
base_module = "",
7777
visibility = [
7878
"//executorch/examples/apple/coreml/...",
7979
"@EXECUTORCH_CLIENTS",
8080
],
81-
external_deps = [
82-
"pybind11",
83-
],
8481
deps = [
8582
"fbsource//third-party/nlohmann-json:nlohmann-json",
83+
"fbsource//third-party/pybind11:pybind11",
8684
],
8785
)
8886

@@ -92,10 +90,10 @@ runtime.python_test(
9290
"test/*.py",
9391
]),
9492
deps = [
93+
"fbsource//third-party/pypi/pytest:pytest",
9594
":partitioner",
9695
":quantizer",
9796
"//caffe2:torch",
9897
"//pytorch/vision:torchvision",
99-
"fbsource//third-party/pypi/pytest:pytest",
10098
],
10199
)

0 commit comments

Comments
 (0)