Alert user about vocab size restrictions (#258)

mozilla · Nov 9, 2023 · 4f47325 · 4f47325
1 parent 2df0a3a
commit 4f47325
Showing 1 changed file with 5 additions and 0 deletions.
diff --git a/pipeline/train/spm-vocab.sh b/pipeline/train/spm-vocab.sh
@@ -15,6 +15,11 @@ sample_size=$4
 threads=$5
 vocab_size="${6:-32000}"
 
+if (( vocab_size % 8 != 0 )); then
+  echo "Error: vocab_size must be a multiple of 8 (https://github.com/mozilla/firefox-translations-training/issues/249)"
+  exit 1
+fi
+
 if [ "$threads" = "auto" ]; then
   threads=$(nproc)
 fi