Skip to content

Commit 0e1ba90

Browse files
manuelcandalesmanuelcandales
committed
Add Metal backend CI workflow with Voxtral testing
1 parent 296e07f commit 0e1ba90

File tree

2 files changed

+233
-1
lines changed

2 files changed

+233
-1
lines changed

‎.ci/docker/ci_commit_pins/optimum-executorch.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
09fdbd0a0639b128f712a4f5202ed42ca4c60957
1+
467660923a5a25e4718e1d6697b93ff1bab4e807

‎.github/workflows/metal.yml‎

Lines changed: 232 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,232 @@
1+
name: Test Metal Backend
2+
3+
on:
4+
pull_request:
5+
push:
6+
branches:
7+
- main
8+
- release/*
9+
10+
concurrency:
11+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}-${{ github.event_name == 'schedule' }}
12+
cancel-in-progress: false
13+
14+
jobs:
15+
test-metal-builds:
16+
name: test-executorch-metal-build
17+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
18+
with:
19+
runner: macos-m2-stable
20+
python-version: '3.11'
21+
submodules: 'recursive'
22+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
23+
timeout: 90
24+
script: |
25+
set -eux
26+
27+
# Print machine info
28+
uname -a
29+
if [ $(uname -s) == Darwin ]; then
30+
sw_vers
31+
# Print RAM in GB
32+
RAM_BYTES=$(sysctl -n hw.memsize)
33+
RAM_GB=$(echo "scale=2; $RAM_BYTES/1024/1024/1024" | bc)
34+
echo "Available RAM (GB): $RAM_GB"
35+
sysctl machdep.cpu.brand_string
36+
sysctl machdep.cpu.core_count
37+
# Print number of GPU cores (Apple Silicon)
38+
if command -v system_profiler &> /dev/null; then
39+
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Total Number of Cores/ {print $5; exit}')
40+
if [ -z "$GPU_CORES" ]; then
41+
# Fallback: try to parse "Core Count" from Apple GPU section
42+
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Core Count/ {print $3; exit}')
43+
fi
44+
echo "GPU Cores: ${GPU_CORES:-Unknown}"
45+
else
46+
echo "system_profiler not available, cannot determine GPU cores."
47+
fi
48+
fi
49+
50+
# Test ExecuTorch Metal build
51+
PYTHON_EXECUTABLE=python CMAKE_ARGS="-DEXECUTORCH_BUILD_METAL=ON" ${CONDA_RUN} --no-capture-output ./install_executorch.sh
52+
53+
export-voxtral-metal-artifact:
54+
name: export-voxtral-metal-artifact
55+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
56+
secrets: inherit
57+
with:
58+
runner: macos-m2-stable
59+
python-version: '3.11'
60+
submodules: 'recursive'
61+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
62+
timeout: 90
63+
secrets-env: EXECUTORCH_HF_TOKEN
64+
upload-artifact: voxtral-metal-export
65+
script: |
66+
set -eux
67+
68+
# Print machine info
69+
uname -a
70+
if [ $(uname -s) == Darwin ]; then
71+
sw_vers
72+
# Print RAM in GB
73+
RAM_BYTES=$(sysctl -n hw.memsize)
74+
RAM_GB=$(echo "scale=2; $RAM_BYTES/1024/1024/1024" | bc)
75+
echo "Available RAM (GB): $RAM_GB"
76+
sysctl machdep.cpu.brand_string
77+
sysctl machdep.cpu.core_count
78+
# Print number of GPU cores (Apple Silicon)
79+
if command -v system_profiler &> /dev/null; then
80+
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Total Number of Cores/ {print $5; exit}')
81+
if [ -z "$GPU_CORES" ]; then
82+
# Fallback: try to parse "Core Count" from Apple GPU section
83+
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Core Count/ {print $3; exit}')
84+
fi
85+
echo "GPU Cores: ${GPU_CORES:-Unknown}"
86+
else
87+
echo "system_profiler not available, cannot determine GPU cores."
88+
fi
89+
fi
90+
91+
echo "::group::Setup Huggingface"
92+
${CONDA_RUN} pip install -U "huggingface_hub[cli]" accelerate
93+
${CONDA_RUN} huggingface-cli login --token $SECRET_EXECUTORCH_HF_TOKEN
94+
95+
echo "::group::Setup Optimum-ExecuTorch"
96+
OPTIMUM_ET_VERSION=$(cat .ci/docker/ci_commit_pins/optimum-executorch.txt)
97+
echo "Using optimum-executorch version: ${OPTIMUM_ET_VERSION}"
98+
${CONDA_RUN} pip install git+https://github.com/huggingface/optimum-executorch.git@${OPTIMUM_ET_VERSION}
99+
${CONDA_RUN} pip install mistral-common librosa
100+
echo "::endgroup::"
101+
102+
echo "::group::Setup ExecuTorch"
103+
PYTHON_EXECUTABLE=python ${CONDA_RUN} ./install_executorch.sh
104+
echo "::endgroup::"
105+
106+
echo "::group::Update transfomers"
107+
${CONDA_RUN} pip install -U transformers
108+
${CONDA_RUN} pip list
109+
echo "::endgroup::"
110+
111+
echo "::group::Export Voxtral"
112+
${CONDA_RUN} optimum-cli export executorch \
113+
--model "mistralai/Voxtral-Mini-3B-2507" \
114+
--task "multimodal-text-to-text" \
115+
--recipe "metal" \
116+
--dtype bfloat16 \
117+
--max_seq_len 1024 \
118+
--output_dir ./
119+
${CONDA_RUN} python -m executorch.extension.audio.mel_spectrogram \
120+
--feature_size 128 \
121+
--stack_output \
122+
--max_audio_len 300 \
123+
--output_file voxtral_preprocessor.pte
124+
125+
test -f model.pte
126+
test -f aoti_metal_blob.ptd
127+
test -f voxtral_preprocessor.pte
128+
echo "::endgroup::"
129+
130+
echo "::group::Store Voxtral Artifacts"
131+
mkdir -p "${RUNNER_ARTIFACT_DIR}"
132+
cp model.pte "${RUNNER_ARTIFACT_DIR}/"
133+
cp aoti_metal_blob.ptd "${RUNNER_ARTIFACT_DIR}/"
134+
cp voxtral_preprocessor.pte "${RUNNER_ARTIFACT_DIR}/"
135+
ls -al "${RUNNER_ARTIFACT_DIR}"
136+
echo "::endgroup::"
137+
138+
test-voxtral-metal-e2e:
139+
name: test-voxtral-metal-e2e
140+
needs: export-voxtral-metal-artifact
141+
uses: pytorch/test-infra/.github/workflows/macos_job.yml@main
142+
with:
143+
runner: macos-m2-stable
144+
python-version: '3.11'
145+
submodules: 'recursive'
146+
ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
147+
timeout: 90
148+
download-artifact: voxtral-metal-export
149+
script: |
150+
set -eux
151+
152+
# Print machine info
153+
uname -a
154+
if [ $(uname -s) == Darwin ]; then
155+
sw_vers
156+
# Print RAM in GB
157+
RAM_BYTES=$(sysctl -n hw.memsize)
158+
RAM_GB=$(echo "scale=2; $RAM_BYTES/1024/1024/1024" | bc)
159+
echo "Available RAM (GB): $RAM_GB"
160+
sysctl machdep.cpu.brand_string
161+
sysctl machdep.cpu.core_count
162+
# Print number of GPU cores (Apple Silicon)
163+
if command -v system_profiler &> /dev/null; then
164+
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Total Number of Cores/ {print $5; exit}')
165+
if [ -z "$GPU_CORES" ]; then
166+
# Fallback: try to parse "Core Count" from Apple GPU section
167+
GPU_CORES=$(system_profiler SPDisplaysDataType | awk '/Core Count/ {print $3; exit}')
168+
fi
169+
echo "GPU Cores: ${GPU_CORES:-Unknown}"
170+
else
171+
echo "system_profiler not available, cannot determine GPU cores."
172+
fi
173+
fi
174+
175+
echo "::group::Setup ExecuTorch Requirements"
176+
CMAKE_ARGS="-DEXECUTORCH_BUILD_METAL=ON" ${CONDA_RUN} --no-capture-output ./install_requirements.sh
177+
${CONDA_RUN} pip list
178+
echo "::endgroup::"
179+
180+
echo "::group::Prepare Voxtral Artifacts"
181+
cp "${RUNNER_ARTIFACT_DIR}/model.pte" .
182+
cp "${RUNNER_ARTIFACT_DIR}/aoti_metal_blob.ptd" .
183+
cp "${RUNNER_ARTIFACT_DIR}/voxtral_preprocessor.pte" .
184+
TOKENIZER_URL="https://huggingface.co/mistralai/Voxtral-Mini-3B-2507/resolve/main/tekken.json"
185+
curl -L $TOKENIZER_URL -o tekken.json
186+
ls -al model.pte aoti_metal_blob.ptd voxtral_preprocessor.pte tekken.json
187+
echo "::endgroup::"
188+
189+
echo "::group::Create Test Audio File"
190+
say -o call_samantha_hall.aiff "Call Samantha Hall"
191+
afconvert -f WAVE -d LEI16 call_samantha_hall.aiff call_samantha_hall.wav
192+
echo "::endgroup::"
193+
194+
echo "::group::Build Voxtral Runner"
195+
cmake --preset llm \
196+
-DEXECUTORCH_BUILD_METAL=ON \
197+
-DCMAKE_INSTALL_PREFIX=cmake-out \
198+
-DCMAKE_BUILD_TYPE=Release \
199+
-Bcmake-out -S.
200+
cmake --build cmake-out -j$(( $(sysctl -n hw.ncpu) - 1 )) --target install --config Release
201+
202+
cmake -DEXECUTORCH_BUILD_METAL=ON \
203+
-DCMAKE_BUILD_TYPE=Release \
204+
-Sexamples/models/voxtral \
205+
-Bcmake-out/examples/models/voxtral/
206+
cmake --build cmake-out/examples/models/voxtral --target voxtral_runner --config Release
207+
echo "::endgroup::"
208+
209+
echo "::group::Run Voxtral Runner"
210+
set +e
211+
OUTPUT=$(cmake-out/examples/models/voxtral/voxtral_runner \
212+
--model_path model.pte \
213+
--data_path aoti_metal_blob.ptd \
214+
--tokenizer_path tekken.json \
215+
--audio_path call_samantha_hall.wav \
216+
--processor_path voxtral_preprocessor.pte \
217+
--temperature 0 2>&1)
218+
EXIT_CODE=$?
219+
set -e
220+
221+
echo "$OUTPUT"
222+
223+
if ! echo "$OUTPUT" | grep -iq "Samantha"; then
224+
echo "Expected output 'Samantha' not found in output"
225+
exit 1
226+
fi
227+
228+
if [ $EXIT_CODE -ne 0 ]; then
229+
echo "Unexpected exit code: $EXIT_CODE"
230+
exit $EXIT_CODE
231+
fi
232+
echo "::endgroup::"

0 commit comments

Comments
 (0)