use --break-system-packages with pip install

now required because of the ubuntu 24.04 upgrade bringing in a newer python
mozilla · Jan 10, 2025 · dc357a1 · dc357a1
1 parent 996ab9e
commit dc357a1
Show file tree

Hide file tree

Showing 35 changed files with 87 additions and 87 deletions.
diff --git a/Taskfile.yml b/Taskfile.yml
@@ -243,7 +243,7 @@ tasks:
     desc: Installs the taskgraph requirements.
     internal: true
     cmds:
-      - poetry run --directory ./taskgraph -- pip3 install -r taskcluster/requirements.txt
+      - poetry run --directory ./taskgraph -- pip3 install --break-system-packages -r taskcluster/requirements.txt
 
   taskgraph-validate:
     desc: Validates Taskcluster task graph locally

diff --git a/docs/tracking.md b/docs/tracking.md
@@ -155,7 +155,7 @@ The parser can be built as a distinct package to make developments easier using
 
 On a virtual environment, you can install the package in editable mode (i.e from the local folder):
 ```sh
-$ pip install -e ./tracking
+$ pip install --break-system-packages -e ./tracking
 ```
 
 ### Extend supported Marian metrics

diff --git a/pipeline/clean/tools/langid_fasttext.py b/pipeline/clean/tools/langid_fasttext.py
@@ -5,7 +5,7 @@
 #   ./langid-fasttext.py < sents.txt > code-tab-sents.txt
 #
 # Installation:
-#   pip3 install pybind11 fasttext --user
+#   pip3 install --break-system-packages pybind11 fasttext --user
 #
 # Parallelize:
 #   cat sents.txt | parallel --pipe -k -j16 --block 20M ./langid-fasttext.py > code-tab-sents.txt

diff --git a/pipeline/setup/install-kenlm.sh b/pipeline/setup/install-kenlm.sh
@@ -21,6 +21,6 @@ cmake .. -DKENLM_MAX_ORDER=7 -DCMAKE_INSTALL_PREFIX:PATH="${BIN}/kenlm"
 make -j "${threads}" install
 cd ..
 
-python -m pip install . --install-option="--max_order 7"
+python -m pip install --break-system-packages . --install-option="--max_order 7"
 
 echo "###### Done: Installing kenlm"
diff --git a/taskcluster/kinds/alignments-backtranslated/kind.yml b/taskcluster/kinds/alignments-backtranslated/kind.yml
@@ -69,8 +69,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt &&
                     export BIN=$MOZ_FETCHES_DIR &&
                     export PATH=$PATH:$MOZ_FETCHES_DIR &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&

diff --git a/taskcluster/kinds/alignments-original/kind.yml b/taskcluster/kinds/alignments-original/kind.yml
@@ -68,8 +68,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt &&
                     export BIN=$MOZ_FETCHES_DIR &&
                     export PATH=$PATH:$MOZ_FETCHES_DIR &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&

diff --git a/taskcluster/kinds/alignments-student/kind.yml b/taskcluster/kinds/alignments-student/kind.yml
@@ -67,8 +67,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt &&
                     export BIN=$MOZ_FETCHES_DIR &&
                     export PATH=$PATH:$MOZ_FETCHES_DIR &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&

diff --git a/taskcluster/kinds/analyze-corpus/kind.yml b/taskcluster/kinds/analyze-corpus/kind.yml
@@ -61,8 +61,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/analyze.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/analyze.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH
                     &&
                     python3 $VCS_PATH/pipeline/data/analyze.py

diff --git a/taskcluster/kinds/analyze-mono/kind.yml b/taskcluster/kinds/analyze-mono/kind.yml
@@ -52,8 +52,8 @@ task-defaults:
             - bash
             - -c
             - >-
-                pip3 install --upgrade pip setuptools &&
-                pip3 install -r $VCS_PATH/pipeline/data/requirements/analyze.txt &&
+                pip3 install --break-system-packages --upgrade pip setuptools &&
+                pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/analyze.txt &&
                 export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                 python3 $VCS_PATH/pipeline/data/analyze.py
                 --file_location $MOZ_FETCHES_DIR/{dataset_sanitized}.{locale}.zst

diff --git a/taskcluster/kinds/bicleaner-model/kind.yml b/taskcluster/kinds/bicleaner-model/kind.yml
@@ -59,9 +59,9 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip install $MOZ_FETCHES_DIR/cyhunspell-2.0.3-cp310-cp310-linux_x86_64.whl &&
-                    pip install $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl &&
-                    pip install -r $VCS_PATH/pipeline/bicleaner/requirements/bicleaner-ai.txt &&
+                    pip install --break-system-packages $MOZ_FETCHES_DIR/cyhunspell-2.0.3-cp310-cp310-linux_x86_64.whl &&
+                    pip install --break-system-packages $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl &&
+                    pip install --break-system-packages -r $VCS_PATH/pipeline/bicleaner/requirements/bicleaner-ai.txt &&
                     export PATH=$PATH:~/.local/bin &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/bicleaner/download_pack.py

diff --git a/taskcluster/kinds/bicleaner/kind.yml b/taskcluster/kinds/bicleaner/kind.yml
@@ -100,9 +100,9 @@ tasks:
                 # 4) number of threads to use - auto means nproc
                 # 5) "pack dir" - which needs to be where the `bicleaner-src-trg` fetch was unpacked to
                 - >-
-                    pip install $MOZ_FETCHES_DIR/cyhunspell-2.0.3-cp310-cp310-linux_x86_64.whl &&
-                    pip install $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl &&
-                    pip install -r {bicleaner_reqs} &&
+                    pip install --break-system-packages $MOZ_FETCHES_DIR/cyhunspell-2.0.3-cp310-cp310-linux_x86_64.whl &&
+                    pip install --break-system-packages $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl &&
+                    pip install --break-system-packages -r {bicleaner_reqs} &&
                     export PATH=$PATH:~/.local/bin &&
                     $VCS_PATH/pipeline/bicleaner/bicleaner.sh
                     $MOZ_FETCHES_DIR/{dataset_sanitized}

diff --git a/taskcluster/kinds/clean-corpus/kind.yml b/taskcluster/kinds/clean-corpus/kind.yml
@@ -98,7 +98,7 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip install -r $VCS_PATH/pipeline/clean/requirements/clean.txt &&
+                    pip install --break-system-packages -r $VCS_PATH/pipeline/clean/requirements/clean.txt &&
                     if [ ${USE_OPUSCLEANER} == "true" ]; then dir="clean/opuscleaner"; else dir="clean"; fi &&
                     $VCS_PATH/pipeline/${dir}/clean-corpus.sh
                     $MOZ_FETCHES_DIR/{dataset_sanitized}

diff --git a/taskcluster/kinds/dataset/kind.yml b/taskcluster/kinds/dataset/kind.yml
@@ -65,8 +65,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/dataset_importer.py
                     --type corpus
@@ -93,8 +93,8 @@ tasks:
                 - bash
                 - -cx
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 -u $VCS_PATH/pipeline/data/dataset_importer.py
                     --type corpus
@@ -122,8 +122,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/dataset_importer.py
                     --type corpus
@@ -151,8 +151,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/dataset_importer.py
                     --type corpus
@@ -186,8 +186,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/download-mono.py
                     --dataset {dataset}
@@ -219,8 +219,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/download-mono.py
                     --dataset {dataset}
@@ -252,8 +252,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/download-mono.py
                     --dataset {dataset}
@@ -285,8 +285,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/download-mono.py
                     --dataset {dataset}
@@ -322,8 +322,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/download-mono.py
                     --dataset {dataset}
@@ -361,8 +361,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/download-mono.py
                     --dataset {dataset}
@@ -389,8 +389,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/dataset_importer.py
                     --type corpus
@@ -424,8 +424,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/download-mono.py
                     --dataset {dataset}
@@ -458,8 +458,8 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/data/download-mono.py
                     --dataset {dataset}

diff --git a/taskcluster/kinds/evaluate-quantized/kind.yml b/taskcluster/kinds/evaluate-quantized/kind.yml
@@ -92,9 +92,9 @@ tasks:
                 - >-
                     export PATH=$PATH:~/.local/bin &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
-                    pip install --upgrade pip &&
-                    pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt &&
-                    pip install $VCS_PATH/tracking &&
+                    pip install --break-system-packages --upgrade pip &&
+                    pip install --break-system-packages -r $VCS_PATH/pipeline/eval/requirements/eval.txt &&
+                    pip install --break-system-packages $VCS_PATH/tracking &&
                     zstd --rm -d $MOZ_FETCHES_DIR/lex.s2t.pruned.zst &&
                     $VCS_PATH/pipeline/eval/eval.py
                     --src               {src_locale}

diff --git a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml
@@ -98,9 +98,9 @@ tasks:
                 - >-
                     export PATH=$PATH:~/.local/bin &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
-                    pip install --upgrade pip &&
-                    pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt &&
-                    pip install $VCS_PATH/tracking &&
+                    pip install --break-system-packages --upgrade pip &&
+                    pip install --break-system-packages -r $VCS_PATH/pipeline/eval/requirements/eval.txt &&
+                    pip install --break-system-packages $VCS_PATH/tracking &&
                     sed -i -e "s,- .*fetches,- $MOZ_FETCHES_DIR," $TASK_WORKDIR/fetches/*.yml &&
                     sed -i -e "s,- .*artifacts,- $MOZ_FETCHES_DIR," $TASK_WORKDIR/fetches/*.yml &&
                     $VCS_PATH/pipeline/eval/eval.py

diff --git a/taskcluster/kinds/evaluate/kind.yml b/taskcluster/kinds/evaluate/kind.yml
@@ -96,9 +96,9 @@ task-defaults:
             - >-
                 export PATH=$PATH:~/.local/bin &&
                 export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
-                pip install --upgrade pip &&
-                pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt &&
-                pip install $VCS_PATH/tracking &&
+                pip install --break-system-packages --upgrade pip &&
+                pip install --break-system-packages -r $VCS_PATH/pipeline/eval/requirements/eval.txt &&
+                pip install --break-system-packages $VCS_PATH/tracking &&
                 sed -i -e "s,- .*fetches,- $MOZ_FETCHES_DIR," $TASK_WORKDIR/fetches/*.yml &&
                 sed -i -e "s,- .*artifacts,- $MOZ_FETCHES_DIR," $TASK_WORKDIR/fetches/*.yml &&
                 $VCS_PATH/pipeline/eval/eval.py

diff --git a/taskcluster/kinds/extract-best/kind.yml b/taskcluster/kinds/extract-best/kind.yml
@@ -82,8 +82,8 @@ tasks:
                 - >-
                     zstd -d --rm $MOZ_FETCHES_DIR/*.zst &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
-                    pip install --upgrade pip &&
-                    pip install -r $VCS_PATH/pipeline/translate/requirements/extract_best.txt &&
+                    pip install --break-system-packages --upgrade pip &&
+                    pip install --break-system-packages -r $VCS_PATH/pipeline/translate/requirements/extract_best.txt &&
                     python3 $VCS_PATH/pipeline/translate/extract_best.py
                     --nbest "$MOZ_FETCHES_DIR/file.{this_chunk}.nbest"
                     --references "$MOZ_FETCHES_DIR/file.{this_chunk}.ref"

diff --git a/taskcluster/kinds/finetune-student/kind.yml b/taskcluster/kinds/finetune-student/kind.yml
@@ -95,9 +95,9 @@ tasks:
                 - bash
                 - -cx
                 - >-
-                    pip3 install --upgrade pip setuptools &&
-                    pip3 install -r $VCS_PATH/pipeline/train/requirements/train.txt &&
-                    pip3 install $VCS_PATH/tracking &&
+                    pip3 install --break-system-packages --upgrade pip setuptools &&
+                    pip3 install --break-system-packages -r $VCS_PATH/pipeline/train/requirements/train.txt &&
+                    pip3 install --break-system-packages $VCS_PATH/tracking &&
                     export PATH="$HOME/.local/bin:$PATH" &&
                     export MARIAN=$MOZ_FETCHES_DIR &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&

diff --git a/taskcluster/kinds/merge-corpus/kind.yml b/taskcluster/kinds/merge-corpus/kind.yml
@@ -83,7 +83,7 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip install -r $VCS_PATH/pipeline/clean/requirements/merge.txt &&
+                    pip install --break-system-packages -r $VCS_PATH/pipeline/clean/requirements/merge.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/clean/merge-corpus.py
                     --src           {src_locale}

diff --git a/taskcluster/kinds/merge-devset/kind.yml b/taskcluster/kinds/merge-devset/kind.yml
@@ -81,7 +81,7 @@ tasks:
                 - bash
                 - -c
                 - >-
-                    pip install -r $VCS_PATH/pipeline/clean/requirements/merge.txt &&
+                    pip install --break-system-packages -r $VCS_PATH/pipeline/clean/requirements/merge.txt &&
                     export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                     python3 $VCS_PATH/pipeline/clean/merge-corpus.py
                     --src           {src_locale}

diff --git a/taskcluster/kinds/merge-mono/kind.yml b/taskcluster/kinds/merge-mono/kind.yml
@@ -77,7 +77,7 @@ task-defaults:
             # 2) max_sentences
             # 3) datasets
             - >-
-                pip install -r $VCS_PATH/pipeline/clean/requirements/merge.txt &&
+                pip install --break-system-packages -r $VCS_PATH/pipeline/clean/requirements/merge.txt &&
                 export PYTHONPATH=$PYTHONPATH:$VCS_PATH &&
                 python3 $VCS_PATH/pipeline/clean/merge-mono.py
                 --parallel_corpus $MOZ_FETCHES_DIR/corpus/corpus.{locale}.zst