From dc357a19df01bc3e9df6d1536a967de10d74e340 Mon Sep 17 00:00:00 2001 From: Ben Hearsum Date: Fri, 10 Jan 2025 10:31:19 -0500 Subject: [PATCH] use --break-system-packages with pip install now required because of the ubuntu 24.04 upgrade bringing in a newer python --- Taskfile.yml | 2 +- docs/tracking.md | 2 +- pipeline/clean/tools/langid_fasttext.py | 2 +- pipeline/setup/install-kenlm.sh | 2 +- .../kinds/alignments-backtranslated/kind.yml | 4 +- .../kinds/alignments-original/kind.yml | 4 +- taskcluster/kinds/alignments-student/kind.yml | 4 +- taskcluster/kinds/analyze-corpus/kind.yml | 4 +- taskcluster/kinds/analyze-mono/kind.yml | 4 +- taskcluster/kinds/bicleaner-model/kind.yml | 6 +-- taskcluster/kinds/bicleaner/kind.yml | 6 +-- taskcluster/kinds/clean-corpus/kind.yml | 2 +- taskcluster/kinds/dataset/kind.yml | 52 +++++++++---------- taskcluster/kinds/evaluate-quantized/kind.yml | 6 +-- .../kinds/evaluate-teacher-ensemble/kind.yml | 6 +-- taskcluster/kinds/evaluate/kind.yml | 6 +-- taskcluster/kinds/extract-best/kind.yml | 4 +- taskcluster/kinds/finetune-student/kind.yml | 6 +-- taskcluster/kinds/merge-corpus/kind.yml | 2 +- taskcluster/kinds/merge-devset/kind.yml | 2 +- taskcluster/kinds/merge-mono/kind.yml | 2 +- taskcluster/kinds/quantize/kind.yml | 2 +- taskcluster/kinds/shortlist/kind.yml | 4 +- taskcluster/kinds/split-corpus/kind.yml | 2 +- taskcluster/kinds/split-mono-src/kind.yml | 2 +- taskcluster/kinds/split-mono-trg/kind.yml | 2 +- taskcluster/kinds/train-backwards/kind.yml | 6 +-- taskcluster/kinds/train-student/kind.yml | 6 +-- taskcluster/kinds/train-teacher/kind.yml | 6 +-- taskcluster/kinds/translate-corpus/kind.yml | 2 +- taskcluster/kinds/translate-mono-src/kind.yml | 2 +- taskcluster/kinds/translate-mono-trg/kind.yml | 2 +- taskcluster/scripts/toolchain/build-kenlm.sh | 2 +- tests/fixtures/__init__.py | 6 +-- utils/tb_log_parser.py | 2 +- 35 files changed, 87 insertions(+), 87 deletions(-) diff --git a/Taskfile.yml b/Taskfile.yml index c76d57b16..46d443c4b 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -243,7 +243,7 @@ tasks: desc: Installs the taskgraph requirements. internal: true cmds: - - poetry run --directory ./taskgraph -- pip3 install -r taskcluster/requirements.txt + - poetry run --directory ./taskgraph -- pip3 install --break-system-packages -r taskcluster/requirements.txt taskgraph-validate: desc: Validates Taskcluster task graph locally diff --git a/docs/tracking.md b/docs/tracking.md index b6d8d4178..fd275f879 100644 --- a/docs/tracking.md +++ b/docs/tracking.md @@ -155,7 +155,7 @@ The parser can be built as a distinct package to make developments easier using On a virtual environment, you can install the package in editable mode (i.e from the local folder): ```sh -$ pip install -e ./tracking +$ pip install --break-system-packages -e ./tracking ``` ### Extend supported Marian metrics diff --git a/pipeline/clean/tools/langid_fasttext.py b/pipeline/clean/tools/langid_fasttext.py index 8adde02d4..986dc3bd8 100755 --- a/pipeline/clean/tools/langid_fasttext.py +++ b/pipeline/clean/tools/langid_fasttext.py @@ -5,7 +5,7 @@ # ./langid-fasttext.py < sents.txt > code-tab-sents.txt # # Installation: -# pip3 install pybind11 fasttext --user +# pip3 install --break-system-packages pybind11 fasttext --user # # Parallelize: # cat sents.txt | parallel --pipe -k -j16 --block 20M ./langid-fasttext.py > code-tab-sents.txt diff --git a/pipeline/setup/install-kenlm.sh b/pipeline/setup/install-kenlm.sh index a566e8b3d..e6ffd2234 100755 --- a/pipeline/setup/install-kenlm.sh +++ b/pipeline/setup/install-kenlm.sh @@ -21,6 +21,6 @@ cmake .. -DKENLM_MAX_ORDER=7 -DCMAKE_INSTALL_PREFIX:PATH="${BIN}/kenlm" make -j "${threads}" install cd .. -python -m pip install . --install-option="--max_order 7" +python -m pip install --break-system-packages . --install-option="--max_order 7" echo "###### Done: Installing kenlm" diff --git a/taskcluster/kinds/alignments-backtranslated/kind.yml b/taskcluster/kinds/alignments-backtranslated/kind.yml index ab32f4737..c8fc33a30 100644 --- a/taskcluster/kinds/alignments-backtranslated/kind.yml +++ b/taskcluster/kinds/alignments-backtranslated/kind.yml @@ -69,8 +69,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && export BIN=$MOZ_FETCHES_DIR && export PATH=$PATH:$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && diff --git a/taskcluster/kinds/alignments-original/kind.yml b/taskcluster/kinds/alignments-original/kind.yml index b59e6db3b..37c4a9e79 100644 --- a/taskcluster/kinds/alignments-original/kind.yml +++ b/taskcluster/kinds/alignments-original/kind.yml @@ -68,8 +68,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && export BIN=$MOZ_FETCHES_DIR && export PATH=$PATH:$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && diff --git a/taskcluster/kinds/alignments-student/kind.yml b/taskcluster/kinds/alignments-student/kind.yml index 7da677231..6e054b8b5 100644 --- a/taskcluster/kinds/alignments-student/kind.yml +++ b/taskcluster/kinds/alignments-student/kind.yml @@ -67,8 +67,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && export BIN=$MOZ_FETCHES_DIR && export PATH=$PATH:$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && diff --git a/taskcluster/kinds/analyze-corpus/kind.yml b/taskcluster/kinds/analyze-corpus/kind.yml index 0232fc959..859ae999a 100644 --- a/taskcluster/kinds/analyze-corpus/kind.yml +++ b/taskcluster/kinds/analyze-corpus/kind.yml @@ -61,8 +61,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/analyze.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/analyze.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/analyze.py diff --git a/taskcluster/kinds/analyze-mono/kind.yml b/taskcluster/kinds/analyze-mono/kind.yml index 667a756ae..465132f2d 100644 --- a/taskcluster/kinds/analyze-mono/kind.yml +++ b/taskcluster/kinds/analyze-mono/kind.yml @@ -52,8 +52,8 @@ task-defaults: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/analyze.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/analyze.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/analyze.py --file_location $MOZ_FETCHES_DIR/{dataset_sanitized}.{locale}.zst diff --git a/taskcluster/kinds/bicleaner-model/kind.yml b/taskcluster/kinds/bicleaner-model/kind.yml index c2caa3659..b78ff4680 100644 --- a/taskcluster/kinds/bicleaner-model/kind.yml +++ b/taskcluster/kinds/bicleaner-model/kind.yml @@ -59,9 +59,9 @@ tasks: - bash - -c - >- - pip install $MOZ_FETCHES_DIR/cyhunspell-2.0.3-cp310-cp310-linux_x86_64.whl && - pip install $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl && - pip install -r $VCS_PATH/pipeline/bicleaner/requirements/bicleaner-ai.txt && + pip install --break-system-packages $MOZ_FETCHES_DIR/cyhunspell-2.0.3-cp310-cp310-linux_x86_64.whl && + pip install --break-system-packages $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl && + pip install --break-system-packages -r $VCS_PATH/pipeline/bicleaner/requirements/bicleaner-ai.txt && export PATH=$PATH:~/.local/bin && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/bicleaner/download_pack.py diff --git a/taskcluster/kinds/bicleaner/kind.yml b/taskcluster/kinds/bicleaner/kind.yml index 282fdcb1f..443a774ce 100644 --- a/taskcluster/kinds/bicleaner/kind.yml +++ b/taskcluster/kinds/bicleaner/kind.yml @@ -100,9 +100,9 @@ tasks: # 4) number of threads to use - auto means nproc # 5) "pack dir" - which needs to be where the `bicleaner-src-trg` fetch was unpacked to - >- - pip install $MOZ_FETCHES_DIR/cyhunspell-2.0.3-cp310-cp310-linux_x86_64.whl && - pip install $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl && - pip install -r {bicleaner_reqs} && + pip install --break-system-packages $MOZ_FETCHES_DIR/cyhunspell-2.0.3-cp310-cp310-linux_x86_64.whl && + pip install --break-system-packages $MOZ_FETCHES_DIR/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl && + pip install --break-system-packages -r {bicleaner_reqs} && export PATH=$PATH:~/.local/bin && $VCS_PATH/pipeline/bicleaner/bicleaner.sh $MOZ_FETCHES_DIR/{dataset_sanitized} diff --git a/taskcluster/kinds/clean-corpus/kind.yml b/taskcluster/kinds/clean-corpus/kind.yml index 04c59230d..1a5ba92fe 100644 --- a/taskcluster/kinds/clean-corpus/kind.yml +++ b/taskcluster/kinds/clean-corpus/kind.yml @@ -98,7 +98,7 @@ tasks: - bash - -c - >- - pip install -r $VCS_PATH/pipeline/clean/requirements/clean.txt && + pip install --break-system-packages -r $VCS_PATH/pipeline/clean/requirements/clean.txt && if [ ${USE_OPUSCLEANER} == "true" ]; then dir="clean/opuscleaner"; else dir="clean"; fi && $VCS_PATH/pipeline/${dir}/clean-corpus.sh $MOZ_FETCHES_DIR/{dataset_sanitized} diff --git a/taskcluster/kinds/dataset/kind.yml b/taskcluster/kinds/dataset/kind.yml index 15cae71f3..0c2f49a51 100644 --- a/taskcluster/kinds/dataset/kind.yml +++ b/taskcluster/kinds/dataset/kind.yml @@ -65,8 +65,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/dataset_importer.py --type corpus @@ -93,8 +93,8 @@ tasks: - bash - -cx - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 -u $VCS_PATH/pipeline/data/dataset_importer.py --type corpus @@ -122,8 +122,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/dataset_importer.py --type corpus @@ -151,8 +151,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/dataset_importer.py --type corpus @@ -186,8 +186,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/download-mono.py --dataset {dataset} @@ -219,8 +219,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/download-mono.py --dataset {dataset} @@ -252,8 +252,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/download-mono.py --dataset {dataset} @@ -285,8 +285,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/download-mono.py --dataset {dataset} @@ -322,8 +322,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/download-mono.py --dataset {dataset} @@ -361,8 +361,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/download-mono.py --dataset {dataset} @@ -389,8 +389,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/dataset_importer.py --type corpus @@ -424,8 +424,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/download-mono.py --dataset {dataset} @@ -458,8 +458,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/data/requirements/data.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/data/requirements/data.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/data/download-mono.py --dataset {dataset} diff --git a/taskcluster/kinds/evaluate-quantized/kind.yml b/taskcluster/kinds/evaluate-quantized/kind.yml index 54dc5fdc8..e7d0806bb 100644 --- a/taskcluster/kinds/evaluate-quantized/kind.yml +++ b/taskcluster/kinds/evaluate-quantized/kind.yml @@ -92,9 +92,9 @@ tasks: - >- export PATH=$PATH:~/.local/bin && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && - pip install --upgrade pip && - pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt && - pip install $VCS_PATH/tracking && + pip install --break-system-packages --upgrade pip && + pip install --break-system-packages -r $VCS_PATH/pipeline/eval/requirements/eval.txt && + pip install --break-system-packages $VCS_PATH/tracking && zstd --rm -d $MOZ_FETCHES_DIR/lex.s2t.pruned.zst && $VCS_PATH/pipeline/eval/eval.py --src {src_locale} diff --git a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml index 3f50496e1..3435625df 100644 --- a/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml +++ b/taskcluster/kinds/evaluate-teacher-ensemble/kind.yml @@ -98,9 +98,9 @@ tasks: - >- export PATH=$PATH:~/.local/bin && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && - pip install --upgrade pip && - pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt && - pip install $VCS_PATH/tracking && + pip install --break-system-packages --upgrade pip && + pip install --break-system-packages -r $VCS_PATH/pipeline/eval/requirements/eval.txt && + pip install --break-system-packages $VCS_PATH/tracking && sed -i -e "s,- .*fetches,- $MOZ_FETCHES_DIR," $TASK_WORKDIR/fetches/*.yml && sed -i -e "s,- .*artifacts,- $MOZ_FETCHES_DIR," $TASK_WORKDIR/fetches/*.yml && $VCS_PATH/pipeline/eval/eval.py diff --git a/taskcluster/kinds/evaluate/kind.yml b/taskcluster/kinds/evaluate/kind.yml index f78484ea7..7c9898734 100644 --- a/taskcluster/kinds/evaluate/kind.yml +++ b/taskcluster/kinds/evaluate/kind.yml @@ -96,9 +96,9 @@ task-defaults: - >- export PATH=$PATH:~/.local/bin && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && - pip install --upgrade pip && - pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt && - pip install $VCS_PATH/tracking && + pip install --break-system-packages --upgrade pip && + pip install --break-system-packages -r $VCS_PATH/pipeline/eval/requirements/eval.txt && + pip install --break-system-packages $VCS_PATH/tracking && sed -i -e "s,- .*fetches,- $MOZ_FETCHES_DIR," $TASK_WORKDIR/fetches/*.yml && sed -i -e "s,- .*artifacts,- $MOZ_FETCHES_DIR," $TASK_WORKDIR/fetches/*.yml && $VCS_PATH/pipeline/eval/eval.py diff --git a/taskcluster/kinds/extract-best/kind.yml b/taskcluster/kinds/extract-best/kind.yml index d65079618..6a12155a5 100644 --- a/taskcluster/kinds/extract-best/kind.yml +++ b/taskcluster/kinds/extract-best/kind.yml @@ -82,8 +82,8 @@ tasks: - >- zstd -d --rm $MOZ_FETCHES_DIR/*.zst && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && - pip install --upgrade pip && - pip install -r $VCS_PATH/pipeline/translate/requirements/extract_best.txt && + pip install --break-system-packages --upgrade pip && + pip install --break-system-packages -r $VCS_PATH/pipeline/translate/requirements/extract_best.txt && python3 $VCS_PATH/pipeline/translate/extract_best.py --nbest "$MOZ_FETCHES_DIR/file.{this_chunk}.nbest" --references "$MOZ_FETCHES_DIR/file.{this_chunk}.ref" diff --git a/taskcluster/kinds/finetune-student/kind.yml b/taskcluster/kinds/finetune-student/kind.yml index fe28d3a62..5ec05aa30 100644 --- a/taskcluster/kinds/finetune-student/kind.yml +++ b/taskcluster/kinds/finetune-student/kind.yml @@ -95,9 +95,9 @@ tasks: - bash - -cx - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/train/requirements/train.txt && - pip3 install $VCS_PATH/tracking && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/train/requirements/train.txt && + pip3 install --break-system-packages $VCS_PATH/tracking && export PATH="$HOME/.local/bin:$PATH" && export MARIAN=$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && diff --git a/taskcluster/kinds/merge-corpus/kind.yml b/taskcluster/kinds/merge-corpus/kind.yml index f4e9a4eb4..57dcd631e 100644 --- a/taskcluster/kinds/merge-corpus/kind.yml +++ b/taskcluster/kinds/merge-corpus/kind.yml @@ -83,7 +83,7 @@ tasks: - bash - -c - >- - pip install -r $VCS_PATH/pipeline/clean/requirements/merge.txt && + pip install --break-system-packages -r $VCS_PATH/pipeline/clean/requirements/merge.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/clean/merge-corpus.py --src {src_locale} diff --git a/taskcluster/kinds/merge-devset/kind.yml b/taskcluster/kinds/merge-devset/kind.yml index 61cfd0005..81c197dc5 100644 --- a/taskcluster/kinds/merge-devset/kind.yml +++ b/taskcluster/kinds/merge-devset/kind.yml @@ -81,7 +81,7 @@ tasks: - bash - -c - >- - pip install -r $VCS_PATH/pipeline/clean/requirements/merge.txt && + pip install --break-system-packages -r $VCS_PATH/pipeline/clean/requirements/merge.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/clean/merge-corpus.py --src {src_locale} diff --git a/taskcluster/kinds/merge-mono/kind.yml b/taskcluster/kinds/merge-mono/kind.yml index f6aa50ad9..5e6d6980e 100644 --- a/taskcluster/kinds/merge-mono/kind.yml +++ b/taskcluster/kinds/merge-mono/kind.yml @@ -77,7 +77,7 @@ task-defaults: # 2) max_sentences # 3) datasets - >- - pip install -r $VCS_PATH/pipeline/clean/requirements/merge.txt && + pip install --break-system-packages -r $VCS_PATH/pipeline/clean/requirements/merge.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/clean/merge-mono.py --parallel_corpus $MOZ_FETCHES_DIR/corpus/corpus.{locale}.zst diff --git a/taskcluster/kinds/quantize/kind.yml b/taskcluster/kinds/quantize/kind.yml index a19de6887..009641987 100644 --- a/taskcluster/kinds/quantize/kind.yml +++ b/taskcluster/kinds/quantize/kind.yml @@ -73,7 +73,7 @@ tasks: export BIN=$MOZ_FETCHES_DIR && export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$MOZ_FETCHES_DIR/cuda-toolkit/lib64" && zstd --rm -d $MOZ_FETCHES_DIR/*.zst && - pip install -r $VCS_PATH/pipeline/quantize/requirements/quantize.txt && + pip install --break-system-packages -r $VCS_PATH/pipeline/quantize/requirements/quantize.txt && mkdir -p scripts/alphas && mv $MOZ_FETCHES_DIR/extract_stats.py scripts/alphas && $VCS_PATH/pipeline/quantize/quantize.sh diff --git a/taskcluster/kinds/shortlist/kind.yml b/taskcluster/kinds/shortlist/kind.yml index 35bc4e6d1..41df6f589 100644 --- a/taskcluster/kinds/shortlist/kind.yml +++ b/taskcluster/kinds/shortlist/kind.yml @@ -68,8 +68,8 @@ tasks: - bash - -c - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && export BIN=$MOZ_FETCHES_DIR && export MARIAN=$MOZ_FETCHES_DIR && export PATH=$PATH:$MOZ_FETCHES_DIR && diff --git a/taskcluster/kinds/split-corpus/kind.yml b/taskcluster/kinds/split-corpus/kind.yml index fea3eed81..2fb8c25d4 100644 --- a/taskcluster/kinds/split-corpus/kind.yml +++ b/taskcluster/kinds/split-corpus/kind.yml @@ -63,7 +63,7 @@ tasks: - bash - -c - >- - pip3 install -r $VCS_PATH/pipeline/translate/requirements/splitter.txt && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/translate/requirements/splitter.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/translate/splitter.py --output_dir=$TASK_WORKDIR/artifacts diff --git a/taskcluster/kinds/split-mono-src/kind.yml b/taskcluster/kinds/split-mono-src/kind.yml index 3356a411f..a09c688d6 100644 --- a/taskcluster/kinds/split-mono-src/kind.yml +++ b/taskcluster/kinds/split-mono-src/kind.yml @@ -62,7 +62,7 @@ task-defaults: - bash - -c - >- - pip3 install -r $VCS_PATH/pipeline/translate/requirements/splitter.txt && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/translate/requirements/splitter.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/translate/splitter.py --output_dir=$TASK_WORKDIR/artifacts diff --git a/taskcluster/kinds/split-mono-trg/kind.yml b/taskcluster/kinds/split-mono-trg/kind.yml index 37ad51514..246b848b1 100644 --- a/taskcluster/kinds/split-mono-trg/kind.yml +++ b/taskcluster/kinds/split-mono-trg/kind.yml @@ -62,7 +62,7 @@ task-defaults: - bash - -c - >- - pip3 install -r $VCS_PATH/pipeline/translate/requirements/splitter.txt && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/translate/requirements/splitter.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/translate/splitter.py --output_dir=$TASK_WORKDIR/artifacts diff --git a/taskcluster/kinds/train-backwards/kind.yml b/taskcluster/kinds/train-backwards/kind.yml index 350184c53..bce94a084 100644 --- a/taskcluster/kinds/train-backwards/kind.yml +++ b/taskcluster/kinds/train-backwards/kind.yml @@ -100,9 +100,9 @@ tasks: - bash - -cx - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/train/requirements/train.txt && - pip3 install $VCS_PATH/tracking && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/train/requirements/train.txt && + pip3 install --break-system-packages $VCS_PATH/tracking && export PATH="$HOME/.local/bin:$PATH" && export MARIAN=$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && diff --git a/taskcluster/kinds/train-student/kind.yml b/taskcluster/kinds/train-student/kind.yml index 1f9db2a23..e3aaa5adb 100644 --- a/taskcluster/kinds/train-student/kind.yml +++ b/taskcluster/kinds/train-student/kind.yml @@ -96,9 +96,9 @@ tasks: - bash - -cx - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/train/requirements/train.txt && - pip3 install $VCS_PATH/tracking && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/train/requirements/train.txt && + pip3 install --break-system-packages $VCS_PATH/tracking && export PATH="$HOME/.local/bin:$PATH" && export MARIAN=$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && diff --git a/taskcluster/kinds/train-teacher/kind.yml b/taskcluster/kinds/train-teacher/kind.yml index 21bde13e9..365eaabc4 100644 --- a/taskcluster/kinds/train-teacher/kind.yml +++ b/taskcluster/kinds/train-teacher/kind.yml @@ -119,9 +119,9 @@ tasks: - bash - -cx - >- - pip3 install --upgrade pip setuptools && - pip3 install -r $VCS_PATH/pipeline/train/requirements/train.txt && - pip3 install $VCS_PATH/tracking && + pip3 install --break-system-packages --upgrade pip setuptools && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/train/requirements/train.txt && + pip3 install --break-system-packages $VCS_PATH/tracking && export PATH="$HOME/.local/bin:$PATH" && export MARIAN=$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && diff --git a/taskcluster/kinds/translate-corpus/kind.yml b/taskcluster/kinds/translate-corpus/kind.yml index 6fe13e969..7ac8648b1 100644 --- a/taskcluster/kinds/translate-corpus/kind.yml +++ b/taskcluster/kinds/translate-corpus/kind.yml @@ -115,7 +115,7 @@ tasks: # double curly braces are used for the chunk substitutions because # this must first be formatted by task-context to get src and trg locale - >- - pip3 install -r $VCS_PATH/pipeline/translate/requirements/translate-ctranslate2.txt && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/translate/requirements/translate-ctranslate2.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/translate/translate.py --input "$MOZ_FETCHES_DIR/file.{{this_chunk}}.zst" diff --git a/taskcluster/kinds/translate-mono-src/kind.yml b/taskcluster/kinds/translate-mono-src/kind.yml index c0d80c052..34d15a8ce 100644 --- a/taskcluster/kinds/translate-mono-src/kind.yml +++ b/taskcluster/kinds/translate-mono-src/kind.yml @@ -119,7 +119,7 @@ tasks: - bash - -xc - >- - pip3 install -r $VCS_PATH/pipeline/translate/requirements/translate-ctranslate2.txt && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/translate/requirements/translate-ctranslate2.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/translate/translate.py --input "$MOZ_FETCHES_DIR/file.{{this_chunk}}.zst" diff --git a/taskcluster/kinds/translate-mono-trg/kind.yml b/taskcluster/kinds/translate-mono-trg/kind.yml index fa7dbbfbd..7a4336f67 100644 --- a/taskcluster/kinds/translate-mono-trg/kind.yml +++ b/taskcluster/kinds/translate-mono-trg/kind.yml @@ -119,7 +119,7 @@ tasks: # double curly braces are used for the chunk substitutions because # this must first be formatted by task-context to get src and trg locale - >- - pip3 install -r $VCS_PATH/pipeline/translate/requirements/translate-ctranslate2.txt && + pip3 install --break-system-packages -r $VCS_PATH/pipeline/translate/requirements/translate-ctranslate2.txt && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && python3 $VCS_PATH/pipeline/translate/translate.py --input "$MOZ_FETCHES_DIR/file.{{this_chunk}}.zst" diff --git a/taskcluster/scripts/toolchain/build-kenlm.sh b/taskcluster/scripts/toolchain/build-kenlm.sh index 13fcb108d..156d60bcc 100755 --- a/taskcluster/scripts/toolchain/build-kenlm.sh +++ b/taskcluster/scripts/toolchain/build-kenlm.sh @@ -13,7 +13,7 @@ KENLM_DIR=$MOZ_FETCHES_DIR/kenlm-source cd $KENLM_DIR # Install these separately so they will install as wheels. # Using `--build-option` below disables wheels even for dependencies. -pip install setuptools wheel cmake +pip install --break-system-packages setuptools wheel cmake MAX_ORDER=7 python3 setup.py bdist_wheel find . cp $KENLM_DIR/dist/kenlm-0.0.0-cp310-cp310-linux_x86_64.whl $UPLOAD_DIR/ diff --git a/tests/fixtures/__init__.py b/tests/fixtures/__init__.py index dfc0effeb..ad35717c7 100644 --- a/tests/fixtures/__init__.py +++ b/tests/fixtures/__init__.py @@ -447,7 +447,7 @@ def find_pipeline_script(commands: Commands) -> str: if len(command_parts) < 2: raise Exception(f"Could not find {script} in: {command}") - # Remove the preamble to the script, which is should be the pip install. + # Remove the preamble to the script, which is should be the pip install --break-system-packages. command_parts[0] = "" # Join the command parts back together to reassemble the command. @@ -458,11 +458,11 @@ def find_requirements(commands: Commands) -> Optional[str]: command = get_command(commands) # Match the following: - # pip install -r $VCS_PATH/pipeline/eval/requirements/eval.txt && ... + # pip install --break-system-packages -r $VCS_PATH/pipeline/eval/requirements/eval.txt && ... # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ match = re.search( r""" - pip3?\ install\ -r\ \$VCS_PATH\/ # Find the pip install. + pip3?\ install\ -r\ \$VCS_PATH\/ # Find the pip install --break-system-packages. (?P # Capture as "requirements" [\w\/\-\.]+ # Match the path ) diff --git a/utils/tb_log_parser.py b/utils/tb_log_parser.py index 12a7e3f6b..aaa3df8ec 100755 --- a/utils/tb_log_parser.py +++ b/utils/tb_log_parser.py @@ -7,7 +7,7 @@ import click # This has a dependency conflict with protobuf versions when poetry installing. -# To run this script run `poetry run pip install tensorboardX`. +# To run this script run `poetry run pip install --break-system-packages tensorboardX`. import tensorboardX as tb # type: ignore