[taskcluster 2024-07-11T12:27:00.167Z] podman cp taskcontainer:/builds/worker/artifacts artifact0
[taskcluster 2024-07-11T12:27:00.167Z] podman rm taskcontainer
[taskcluster 2024-07-11T12:27:00.167Z] exit "${exit_code}"'
++ pwd
+ timeout 604800 podman run -t --name taskcontainer --privileged -v /task_172070071020138/cache0:/builds/worker/checkouts --add-host=taskcluster:127.0.0.1 --net=host -e ARTIFACT_EXT -e COMPRESSION_CMD -e FIREFOX_TRANSLATIONS_TRAINING_BASE_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REF -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REV -e FIREFOX_TRANSLATIONS_TRAINING_REPOSITORY_TYPE -e HG_STORE_PATH -e MOZ_AUTOMATION -e MOZ_FETCHES -e MOZ_FETCHES_DIR -e MOZ_SCM_LEVEL -e REPOSITORIES -e RUN_ID -e SCCACHE_DISABLE -e SRC -e TASKCLUSTER_CACHES -e TASKCLUSTER_PROXY_URL -e TASKCLUSTER_ROOT_URL -e TASKCLUSTER_UNTRUSTED_CACHES -e TASKCLUSTER_VOLUMES -e TASKCLUSTER_WORKER_LOCATION -e TASK_ID -e TRG -e VCS_PATH docker-archive:dockerimage /usr/local/bin/run-task --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c 'pip3 install --upgrade pip setuptools && pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && export BIN=$MOZ_FETCHES_DIR && export MARIAN=$MOZ_FETCHES_DIR && export PATH=$PATH:$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && $VCS_PATH/pipeline/alignments/generate-shortlist.sh $MOZ_FETCHES_DIR/corpus $MOZ_FETCHES_DIR/vocab.spm $TASK_WORKDIR/artifacts auto'
Getting image source signatures
Copying blob sha256:1a84dc453bda5bdb00677e6e599015475a779eb490b8b3158be45af99da083e3
Copying blob sha256:50b43097ddf2501adc2f8b21b8685a7508ddd563177028cae61a8ce8cd51a0bb
Copying blob sha256:5498e8c22f6996f25ef193ee58617d5b37e2a96decf22e72de13c3b34e147591
Copying config sha256:1e62b245ce9b86147f4b5a4cdcf3708c9a763f8cdaa58df95c74f81427d083ab
Writing manifest to image destination
Storing signatures
[setup 2024-07-11T12:27:05.965Z] run-task started in /builds/worker
[setup 2024-07-11T12:27:05.965Z] Invoked by command: --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c pip3 install --upgrade pip setuptools && pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && export BIN=$MOZ_FETCHES_DIR && export MARIAN=$MOZ_FETCHES_DIR && export PATH=$PATH:$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && $VCS_PATH/pipeline/alignments/generate-shortlist.sh $MOZ_FETCHES_DIR/corpus $MOZ_FETCHES_DIR/vocab.spm $TASK_WORKDIR/artifacts auto
[setup 2024-07-11T12:27:05.965Z] Python version: 3.10.12
[cache 2024-07-11T12:27:05.967Z] cache /builds/worker/checkouts is empty; writing requirements: gid=1000 uid=1000 version=1
[volume 2024-07-11T12:27:05.967Z] volume /builds/worker/checkouts is a cache
[setup 2024-07-11T12:27:05.967Z] running as worker:worker
[vcs 2024-07-11T12:27:05.967Z] executing ['git', 'config', '--global', '--add', 'safe.directory', '/builds/worker/checkouts/vcs']
[vcs 2024-07-11T12:27:05.969Z] executing ['git', 'clone', 'https://github.com/mozilla/firefox-translations-training', '/builds/worker/checkouts/vcs']
[vcs 2024-07-11T12:27:05.970Z] Cloning into '/builds/worker/checkouts/vcs'...
[vcs 2024-07-11T12:27:06.909Z] executing ['git', 'fetch', '--tags', '--force', 'https://github.com/mozilla/firefox-translations-training', 'release_stages']
[vcs 2024-07-11T12:27:07.097Z] From https://github.com/mozilla/firefox-translations-training
[vcs 2024-07-11T12:27:07.097Z]  * branch            release_stages -> FETCH_HEAD
[vcs 2024-07-11T12:27:07.102Z] executing ['git', 'fetch', '--no-tags', 'https://github.com/mozilla/firefox-translations-training', 'release_stages']
[vcs 2024-07-11T12:27:07.273Z] From https://github.com/mozilla/firefox-translations-training
[vcs 2024-07-11T12:27:07.273Z]  * branch            release_stages -> FETCH_HEAD
[vcs 2024-07-11T12:27:07.279Z] executing ['git', 'checkout', '-f', '-B', 'release_stages', '5ee461fb08b391127e8b58fdacee8a9f4dacf87f']
[vcs 2024-07-11T12:27:07.346Z] Switched to a new branch 'release_stages'
[vcs 2024-07-11T12:27:07.347Z] executing ['git', 'submodule', 'init']
[vcs 2024-07-11T12:27:07.373Z] Submodule '3rd_party/browsermt-marian-dev' (https://github.com/browsermt/marian-dev) registered for path '3rd_party/browsermt-marian-dev'
[vcs 2024-07-11T12:27:07.374Z] Submodule 'extract-lex' (https://github.com/marian-nmt/extract-lex) registered for path '3rd_party/extract-lex'
[vcs 2024-07-11T12:27:07.374Z] Submodule 'fast_align' (https://github.com/clab/fast_align) registered for path '3rd_party/fast_align'
[vcs 2024-07-11T12:27:07.375Z] Submodule '3rd_party/kenlm' (https://github.com/kpu/kenlm) registered for path '3rd_party/kenlm'
[vcs 2024-07-11T12:27:07.375Z] Submodule '3rd_party/marian-dev' (https://github.com/marian-nmt/marian-dev) registered for path '3rd_party/marian-dev'
[vcs 2024-07-11T12:27:07.376Z] Submodule '3rd_party/preprocess' (https://github.com/kpu/preprocess.git) registered for path '3rd_party/preprocess'
[vcs 2024-07-11T12:27:07.376Z] executing ['git', 'submodule', 'update', '--force']
[vcs 2024-07-11T12:27:07.397Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/browsermt-marian-dev'...
[vcs 2024-07-11T12:27:08.425Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/extract-lex'...
[vcs 2024-07-11T12:27:08.763Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/fast_align'...
[vcs 2024-07-11T12:27:09.048Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/kenlm'...
[vcs 2024-07-11T12:27:09.623Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/marian-dev'...
[vcs 2024-07-11T12:27:10.788Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/preprocess'...
[vcs 2024-07-11T12:27:11.251Z] Submodule path '3rd_party/browsermt-marian-dev': checked out '11c6ae7c46be21ef96ed10c60f28022fa968939f'
[vcs 2024-07-11T12:27:11.263Z] Submodule path '3rd_party/extract-lex': checked out '42fa605b53f32eaf6c6e0b5677255c21c91b3d49'
[vcs 2024-07-11T12:27:11.276Z] Submodule path '3rd_party/fast_align': checked out 'cab1e9aac8d3bb02ff5ae58218d8d225a039fa11'
[vcs 2024-07-11T12:27:11.302Z] Submodule path '3rd_party/kenlm': checked out 'bbf4fc511266c5d4515047055d7bdec659a6e158'
[vcs 2024-07-11T12:27:11.401Z] Submodule path '3rd_party/marian-dev': checked out 'e8a1a2530fb84cbff7383302ebca393e5875c441'
[vcs 2024-07-11T12:27:11.421Z] Submodule path '3rd_party/preprocess': checked out '64307314b4d5a9a0bd529b5c1036b0710d995eec'
[vcs 2024-07-11T12:27:11.421Z] cleaning git checkout...
[vcs 2024-07-11T12:27:11.421Z] executing ['git', 'clean', '-nxdff']
[vcs 2024-07-11T12:27:11.424Z] removing []
[vcs 2024-07-11T12:27:11.424Z] successfully cleaned git checkout!
[vcs 2024-07-11T12:27:11.425Z] TinderboxPrint:<a href='https://github.com/mozilla/firefox-translations-training/commit/5ee461fb08b391127e8b58fdacee8a9f4dacf87f' title='Built from firefox-translations-training commit 5ee461fb08b391127e8b58fdacee8a9f4dacf87f'>5ee461fb08b391127e8b58fdacee8a9f4dacf87f</a>
[setup 2024-07-11T12:27:11.425Z] MOZ_FETCHES_DIR is /builds/worker/fetches
[fetches 2024-07-11T12:27:11.425Z] fetching artifacts
[fetches 2024-07-11T12:27:11.425Z] executing ['/usr/bin/python3', '-u', '/usr/local/bin/fetch-content', 'task-artifacts']
attempt 1/5
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/WYdAWxmRS8a6ra1--sk2mQ/artifacts/public/build/corpus.en.zst to /builds/worker/fetches/corpus.en.zst
attempt 1/5
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/WYdAWxmRS8a6ra1--sk2mQ/artifacts/public/build/corpus.ru.zst to /builds/worker/fetches/corpus.ru.zstDownloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/WYdAWxmRS8a6ra1--sk2mQ/artifacts/public/build/corpus.en.zst

attempt 1/5
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/VF--UyYRTI6hby6Gr143Vw/artifacts/public/build/extract_lex.tar.zst to /builds/worker/fetches/extract_lex.tar.zst
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/WYdAWxmRS8a6ra1--sk2mQ/artifacts/public/build/corpus.ru.zst
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/VF--UyYRTI6hby6Gr143Vw/artifacts/public/build/extract_lex.tar.zst
attempt 1/5
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/QYRWW9BSTh-34mlpXqVVsg/artifacts/public/build/fast-align.tar.zst to /builds/worker/fetches/fast-align.tar.zst
attempt 1/5Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/QYRWW9BSTh-34mlpXqVVsg/artifacts/public/build/fast-align.tar.zst

Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/cCr_z2sITDGiow53WiHs8g/artifacts/public/build/marian.tar.zst to /builds/worker/fetches/marian.tar.zstattempt 1/5

Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/Qb5OxVBiRJuXibORs6iO6w/artifacts/public/build/vocab.spm to /builds/worker/fetches/vocab.spm
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/cCr_z2sITDGiow53WiHs8g/artifacts/public/build/marian.tar.zstDownloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/Qb5OxVBiRJuXibORs6iO6w/artifacts/public/build/vocab.spm

https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/WYdAWxmRS8a6ra1--sk2mQ/artifacts/public/build/corpus.ru.zst resolved to 955465 bytes with sha256 f386a1c3a1e4b66612bd627663634cc9eb89c65e327f251c3871f01cd3645d90 in 0.142s
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/WYdAWxmRS8a6ra1--sk2mQ/artifacts/public/build/corpus.en.zst resolved to 174985 bytes with sha256 158e9653be8eff98b12d4f19c1159ac505fb700b376d2ffd5ee7e0582af7c6bc in 0.192s
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/QYRWW9BSTh-34mlpXqVVsg/artifacts/public/build/fast-align.tar.zst resolved to 722791 bytes with sha256 0cd2223cd464783e79948ea5057aa6e482f3f0bbe6660d3cd9200fab91f0e409 in 0.211s
Extracting /builds/worker/fetches/fast-align.tar.zst to /builds/worker/fetches
/builds/worker/fetches/fast-align.tar.zst extracted in 0.006s
Removing /builds/worker/fetches/fast-align.tar.zst
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/VF--UyYRTI6hby6Gr143Vw/artifacts/public/build/extract_lex.tar.zst resolved to 620577 bytes with sha256 964eea4e00f3761a1fc0cc3d50b562a2bbcfdd9b7f668c38d4e2f3ed12c0d9c1 in 0.230s
Extracting /builds/worker/fetches/extract_lex.tar.zst to /builds/worker/fetches
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/Qb5OxVBiRJuXibORs6iO6w/artifacts/public/build/vocab.spm resolved to 252592 bytes with sha256 cfe92f74764c406148110c47da747b23f4b84227eb0f9baed40647160bd81427 in 0.233s
/builds/worker/fetches/extract_lex.tar.zst extracted in 0.006s
Removing /builds/worker/fetches/extract_lex.tar.zst
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/cCr_z2sITDGiow53WiHs8g/artifacts/public/build/marian.tar.zst resolved to 684170918 bytes with sha256 674b3ad4462d1b7ef22e1f2a5a1e340385136821b0e5f4b48af357281a41d819 in 7.226s
Extracting /builds/worker/fetches/marian.tar.zst to /builds/worker/fetches
/builds/worker/fetches/marian.tar.zst extracted in 4.543s
Removing /builds/worker/fetches/marian.tar.zst
PERFHERDER_DATA: {"framework": {"name": "build_metrics"}, "suites": [{"name": "fetch_content", "value": 11.845249558000006, "lowerIsBetter": true, "shouldAlert": false, "subtests": []}]}
[fetches 2024-07-11T12:27:23.341Z] finished fetching artifacts
[task 2024-07-11T12:27:23.341Z] executing ['bash', '-c', 'pip3 install --upgrade pip setuptools && pip3 install -r $VCS_PATH/pipeline/alignments/requirements/alignments.txt && export BIN=$MOZ_FETCHES_DIR && export MARIAN=$MOZ_FETCHES_DIR && export PATH=$PATH:$MOZ_FETCHES_DIR && export PYTHONPATH=$PYTHONPATH:$VCS_PATH && $VCS_PATH/pipeline/alignments/generate-shortlist.sh $MOZ_FETCHES_DIR/corpus $MOZ_FETCHES_DIR/vocab.spm $TASK_WORKDIR/artifacts auto']
[task 2024-07-11T12:27:23.606Z] WARNING: The directory '/builds/worker/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.
[task 2024-07-11T12:27:23.606Z] Defaulting to user installation because normal site-packages is not writeable
[task 2024-07-11T12:27:23.625Z] Requirement already satisfied: pip in /usr/lib/python3/dist-packages (22.0.2)
[task 2024-07-11T12:27:23.770Z] Collecting pip
[task 2024-07-11T12:27:23.943Z]   Downloading pip-24.1.2-py3-none-any.whl (1.8 MB)
[task 2024-07-11T12:27:24.122Z]      ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.8/1.8 MB 10.4 MB/s eta 0:00:00
[task 2024-07-11T12:27:24.131Z] Requirement already satisfied: setuptools in /usr/lib/python3/dist-packages (59.6.0)
[task 2024-07-11T12:27:24.368Z] Collecting setuptools
[task 2024-07-11T12:27:24.405Z]   Downloading setuptools-70.3.0-py3-none-any.whl (931 kB)
[task 2024-07-11T12:27:24.412Z]      ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 931.1/931.1 KB 157.4 MB/s eta 0:00:00
[task 2024-07-11T12:27:24.490Z] Installing collected packages: setuptools, pip
[task 2024-07-11T12:27:26.623Z] Successfully installed pip-24.1.2 setuptools-70.3.0
[task 2024-07-11T12:27:26.921Z] WARNING: The directory '/builds/worker/.cache/pip' or its parent directory is not owned or is not writable by the current user. The cache has been disabled. Check the permissions and owner of that directory. If executing pip with sudo, you should use sudo's -H flag.
[task 2024-07-11T12:27:26.922Z] Defaulting to user installation because normal site-packages is not writeable
[task 2024-07-11T12:27:27.231Z] Collecting cython==3.0.9 (from -r /builds/worker/checkouts/vcs/pipeline/alignments/requirements/alignments.txt (line 7))
[task 2024-07-11T12:27:27.333Z]   Downloading Cython-3.0.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.2 kB)
[task 2024-07-11T12:27:27.372Z] Collecting eflomal==1.0.0b1 (from -r /builds/worker/checkouts/vcs/pipeline/alignments/requirements/alignments.txt (line 9))
[task 2024-07-11T12:27:27.406Z]   Downloading eflomal-1.0.0b1.tar.gz (102 kB)
[task 2024-07-11T12:27:27.472Z]      ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 102.0/102.0 kB 1.5 MB/s eta 0:00:00
[task 2024-07-11T12:27:27.484Z]   Installing build dependencies: started
[task 2024-07-11T12:27:34.806Z]   Installing build dependencies: finished with status 'done'
[task 2024-07-11T12:27:34.807Z]   Getting requirements to build wheel: started
[task 2024-07-11T12:27:36.135Z]   Getting requirements to build wheel: finished with status 'done'
[task 2024-07-11T12:27:36.136Z]   Preparing metadata (pyproject.toml): started
[task 2024-07-11T12:27:36.649Z]   Preparing metadata (pyproject.toml): finished with status 'done'
[task 2024-07-11T12:27:36.809Z] Collecting numpy==1.26.4 (from -r /builds/worker/checkouts/vcs/pipeline/alignments/requirements/alignments.txt (line 11))
[task 2024-07-11T12:27:36.843Z]   Downloading numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (61 kB)
[task 2024-07-11T12:27:36.851Z]      ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 61.0/61.0 kB 8.8 MB/s eta 0:00:00
[task 2024-07-11T12:27:36.895Z] Collecting opus-fast-mosestokenizer==0.0.8.5 (from -r /builds/worker/checkouts/vcs/pipeline/alignments/requirements/alignments.txt (line 13))
[task 2024-07-11T12:27:36.930Z]   Downloading opus_fast_mosestokenizer-0.0.8.5-cp310-cp310-manylinux1_x86_64.whl.metadata (3.6 kB)
[task 2024-07-11T12:27:36.987Z] Collecting tqdm==4.66.4 (from -r /builds/worker/checkouts/vcs/pipeline/alignments/requirements/alignments.txt (line 15))
[task 2024-07-11T12:27:37.021Z]   Downloading tqdm-4.66.4-py3-none-any.whl.metadata (57 kB)
[task 2024-07-11T12:27:37.029Z]      ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 57.6/57.6 kB 8.9 MB/s eta 0:00:00
[task 2024-07-11T12:27:37.035Z] WARNING: The candidate selected for download or install is a yanked version: 'eflomal' candidate (version 1.0.0b1 at https://files.pythonhosted.org/packages/1d/cc/2cdf9c90a6bd8fd7323b47bf5a60fe738bfe8465844a2136b0543f5094e8/eflomal-1.0.0b1.tar.gz (from https://pypi.org/simple/eflomal/))
[task 2024-07-11T12:27:37.035Z] Reason for being yanked: <none given>
[task 2024-07-11T12:27:37.069Z] Downloading Cython-3.0.9-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.6 MB)
[task 2024-07-11T12:27:37.218Z]    ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.6/3.6 MB 24.7 MB/s eta 0:00:00
[task 2024-07-11T12:27:37.252Z] Downloading numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (18.2 MB)
[task 2024-07-11T12:27:37.424Z]    ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 18.2/18.2 MB 102.6 MB/s eta 0:00:00
[task 2024-07-11T12:27:37.458Z] Downloading opus_fast_mosestokenizer-0.0.8.5-cp310-cp310-manylinux1_x86_64.whl (837 kB)
[task 2024-07-11T12:27:37.462Z]    ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 837.2/837.2 kB 396.4 MB/s eta 0:00:00
[task 2024-07-11T12:27:37.496Z] Downloading tqdm-4.66.4-py3-none-any.whl (78 kB)
[task 2024-07-11T12:27:37.498Z]    ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 78.3/78.3 kB 447.5 MB/s eta 0:00:00
[task 2024-07-11T12:27:37.571Z] Building wheels for collected packages: eflomal
[task 2024-07-11T12:27:37.572Z]   Building wheel for eflomal (pyproject.toml): started
[task 2024-07-11T12:27:41.993Z]   Building wheel for eflomal (pyproject.toml): finished with status 'done'
[task 2024-07-11T12:27:41.995Z]   Created wheel for eflomal: filename=eflomal-1.0.0b1-cp310-cp310-linux_x86_64.whl size=319817 sha256=b1354ef4130f54e28cf2318dd7a2cd2157b4bda0843875568c382b1c04d43db7
[task 2024-07-11T12:27:41.995Z]   Stored in directory: /tmp/pip-ephem-wheel-cache-1cme11aw/wheels/b4/0c/55/4a2c6f00793a0d2fdaef4376b8cb3d9c6ae8df16366d6e31d1
[task 2024-07-11T12:27:41.997Z] Successfully built eflomal
[task 2024-07-11T12:27:42.024Z] Installing collected packages: tqdm, opus-fast-mosestokenizer, numpy, cython, eflomal
[task 2024-07-11T12:27:46.629Z] Successfully installed cython-3.0.9 eflomal-1.0.0b1 numpy-1.26.4 opus-fast-mosestokenizer-0.0.8.5 tqdm-4.66.4
[task 2024-07-11T12:27:46.836Z] + set -euo pipefail
[task 2024-07-11T12:27:46.836Z] + echo '###### Generating alignments and shortlist'
[task 2024-07-11T12:27:46.836Z] ###### Generating alignments and shortlist
[task 2024-07-11T12:27:46.837Z] + [[ -z /builds/worker/fetches ]]
[task 2024-07-11T12:27:46.837Z] + [[ -z /builds/worker/fetches ]]
[task 2024-07-11T12:27:46.837Z] + [[ -z ru ]]
[task 2024-07-11T12:27:46.837Z] + [[ -z en ]]
[task 2024-07-11T12:27:46.837Z] + corpus_prefix=/builds/worker/fetches/corpus
[task 2024-07-11T12:27:46.837Z] + vocab_path=/builds/worker/fetches/vocab.spm
[task 2024-07-11T12:27:46.837Z] + output_dir=/builds/worker/artifacts
[task 2024-07-11T12:27:46.837Z] + threads=auto
[task 2024-07-11T12:27:46.837Z] + COMPRESSION_CMD=zstdmt
[task 2024-07-11T12:27:46.837Z] + ARTIFACT_EXT=zst
[task 2024-07-11T12:27:46.837Z] + '[' auto = auto ']'
[task 2024-07-11T12:27:46.837Z] ++ nproc
[task 2024-07-11T12:27:46.837Z] + threads=64
[task 2024-07-11T12:27:46.838Z] ++ dirname /builds/worker/checkouts/vcs/pipeline/alignments/generate-shortlist.sh
[task 2024-07-11T12:27:46.838Z] + cd /builds/worker/checkouts/vcs/pipeline/alignments
[task 2024-07-11T12:27:46.838Z] + mkdir -p /builds/worker/artifacts
[task 2024-07-11T12:27:46.839Z] + dir=/builds/worker/artifacts/tmp
[task 2024-07-11T12:27:46.839Z] + mkdir -p /builds/worker/artifacts/tmp
[task 2024-07-11T12:27:46.840Z] + corpus_src=/builds/worker/fetches/corpus.ru.zst
[task 2024-07-11T12:27:46.840Z] + corpus_trg=/builds/worker/fetches/corpus.en.zst
[task 2024-07-11T12:27:46.840Z] + echo '### Subword segmentation with SentencePiece'
[task 2024-07-11T12:27:46.840Z] ### Subword segmentation with SentencePiece
[task 2024-07-11T12:27:46.841Z] + zstdmt -dc /builds/worker/fetches/corpus.ru.zst
[task 2024-07-11T12:27:46.841Z] + parallel --no-notice --pipe -k -j 64 --block 50M /builds/worker/fetches/spm_encode --model /builds/worker/fetches/vocab.spm
[task 2024-07-11T12:27:47.609Z] + zstdmt -dc /builds/worker/fetches/corpus.en.zst
[task 2024-07-11T12:27:47.609Z] + parallel --no-notice --pipe -k -j 64 --block 50M /builds/worker/fetches/spm_encode --model /builds/worker/fetches/vocab.spm
[task 2024-07-11T12:27:48.822Z] + python3 align.py --corpus_src=/builds/worker/artifacts/tmp/corpus.spm.ru --corpus_trg=/builds/worker/artifacts/tmp/corpus.spm.en --output_path=/builds/worker/artifacts/corpus.aln
[task 2024-07-11T12:27:48.867Z] [alignments] Running with arguments: ['align.py', '--corpus_src=/builds/worker/artifacts/tmp/corpus.spm.ru', '--corpus_trg=/builds/worker/artifacts/tmp/corpus.spm.en', '--output_path=/builds/worker/artifacts/corpus.aln']
[task 2024-07-11T12:27:48.869Z] [alignments] Starting generating alignments.
[task 2024-07-11T12:27:48.942Z] [alignments] Calculating alignments...
[task 2024-07-11T12:27:49.640Z] [eflomal] Prepared 16044 sentences for alignment
[task 2024-07-11T12:27:49.640Z] /builds/worker/.local/lib/python3.10/site-packages/eflomal/bin/eflomal -m 3 -s /tmp/tmpvn8im9tn -t /tmp/tmpn513nac3 -n 3 -N 0.2 -1 9 -2 9 -3 39 -f /builds/worker/artifacts/tmp/aln.fwd -r /builds/worker/artifacts/tmp/aln.rev
[task 2024-07-11T12:27:49.824Z] Read texts (16044 sentences): 0.183 s
[task 2024-07-11T12:27:49.824Z] Vocabulary sizes are 514 (source), 90 (target)
[task 2024-07-11T12:27:49.827Z] Created alignment structures: 0.001 s
[task 2024-07-11T12:27:49.827Z] Created alignment structures: 0.001 s
[task 2024-07-11T12:27:49.842Z] Randomized alignment: 0.015 s
[task 2024-07-11T12:27:49.843Z] Aligning with model 1 (9 iterations)
[task 2024-07-11T12:27:49.857Z] Randomized alignment: 0.029 s
[task 2024-07-11T12:27:49.857Z] Aligning with model 1 (9 iterations)
[task 2024-07-11T12:27:59.217Z] Done: 9.374 s
[task 2024-07-11T12:27:59.217Z] Aligning with model 2 (9 iterations)
[task 2024-07-11T12:27:59.684Z] Done: 9.828 s
[task 2024-07-11T12:27:59.685Z] Aligning with model 2 (9 iterations)
[task 2024-07-11T12:28:08.827Z] Done: 9.610 s
[task 2024-07-11T12:28:08.827Z] Aligning with model 3 (39 iterations)
[task 2024-07-11T12:28:10.001Z] Done: 10.316 s
[task 2024-07-11T12:28:10.001Z] Aligning with model 3 (39 iterations)
[task 2024-07-11T12:28:54.610Z] Done: 45.783 s
[task 2024-07-11T12:28:58.221Z] Done: 48.220 s
[task 2024-07-11T12:28:58.781Z] Final argmax iteration: 4.171 s
[task 2024-07-11T12:28:58.781Z] Writing alignments to /builds/worker/artifacts/tmp/aln.rev for 16044 sentencess
[task 2024-07-11T12:29:02.555Z] Final argmax iteration: 4.333 s
[task 2024-07-11T12:29:02.555Z] Writing alignments to /builds/worker/artifacts/tmp/aln.fwd for 16044 sentencess
[task 2024-07-11T12:29:02.713Z] [alignments] Symmetrizing alignments...
[task 2024-07-11T12:29:05.507Z] [alignments] Finished generating alignments.
[task 2024-07-11T12:29:05.523Z] + echo '### Creating shortlist'
[task 2024-07-11T12:29:05.523Z] ### Creating shortlist
[task 2024-07-11T12:29:05.523Z] + /builds/worker/fetches/extract_lex /builds/worker/artifacts/tmp/corpus.spm.en /builds/worker/artifacts/tmp/corpus.spm.ru /builds/worker/artifacts/corpus.aln /builds/worker/artifacts/tmp/lex.s2t /builds/worker/artifacts/tmp/lex.t2s
[task 2024-07-11T12:29:05.525Z] Starting...
[task 2024-07-11T12:29:06.108Z] 0 10000 
[task 2024-07-11T12:29:06.108Z] Finished
[task 2024-07-11T12:29:06.109Z] + '[' -f /builds/worker/artifacts/tmp/lex.s2t ']'
[task 2024-07-11T12:29:06.109Z] + zstdmt /builds/worker/artifacts/tmp/lex.s2t
[task 2024-07-11T12:29:06.111Z] 
[task 2024-07-11T12:29:06.111Z]                                                                                
[task 2024-07-11T12:29:06.111Z] Read :  0 /  0 MB ==> 33%
[task 2024-07-11T12:29:06.111Z]                                                                                
[task 2024-07-11T12:29:06.111Z] /builds/worker/artifacts/tmp/lex.s2t : 32.60%   ( 89074 =>  29038 bytes, /builds/worker/artifacts/tmp/lex.s2t.zst) 
[task 2024-07-11T12:29:06.112Z] + rm /builds/worker/artifacts/tmp/corpus.spm.en
[task 2024-07-11T12:29:06.115Z] + rm /builds/worker/artifacts/tmp/corpus.spm.ru
[task 2024-07-11T12:29:06.116Z] + rm /builds/worker/artifacts/corpus.aln
[task 2024-07-11T12:29:06.117Z] + echo '### Shortlist pruning'
[task 2024-07-11T12:29:06.117Z] ### Shortlist pruning
[task 2024-07-11T12:29:06.117Z] + /builds/worker/fetches/spm_export_vocab --model=/builds/worker/fetches/vocab.spm --output=/builds/worker/artifacts/tmp/vocab.txt
[task 2024-07-11T12:29:06.121Z] + zstdmt -dc /builds/worker/artifacts/tmp/lex.s2t.zst
[task 2024-07-11T12:29:06.121Z] + grep -v NULL
[task 2024-07-11T12:29:06.122Z] + python3 prune_shortlist.py 100 /builds/worker/artifacts/tmp/vocab.txt
[task 2024-07-11T12:29:06.122Z] + zstdmt
[task 2024-07-11T12:29:06.146Z] + echo '### Deleting tmp dir'
[task 2024-07-11T12:29:06.146Z] ### Deleting tmp dir
[task 2024-07-11T12:29:06.146Z] + rm -rf /builds/worker/artifacts/tmp
[task 2024-07-11T12:29:06.149Z] + echo '###### Done: Generating alignments and shortlist'
[task 2024-07-11T12:29:06.149Z] ###### Done: Generating alignments and shortlist
[fetches 2024-07-11T12:29:06.150Z] removing /builds/worker/fetches
[fetches 2024-07-11T12:29:06.373Z] finished
+ exit_code=0
+ podman cp taskcontainer:/builds/worker/artifacts artifact0
+ podman rm taskcontainer
ff6fb21602d19d1fa4a0f9e79f6f16d2f52cd0758c97d4d8b370553554d55635
+ exit 0
[taskcluster 2024-07-11T12:29:07.273Z]    Exit Code: 0
[taskcluster 2024-07-11T12:29:07.273Z]    User Time: 8.040601s
[taskcluster 2024-07-11T12:29:07.273Z]  Kernel Time: 3.508978s
[taskcluster 2024-07-11T12:29:07.273Z]    Wall Time: 2m7.105652764s
[taskcluster 2024-07-11T12:29:07.273Z]       Result: SUCCEEDED
[taskcluster 2024-07-11T12:29:07.273Z] === Task Finished ===
[taskcluster 2024-07-11T12:29:07.273Z] Task Duration: 2m7.106165542s
[taskcluster 2024-07-11T12:29:07.375Z] Uploading artifact public/build/lex.s2t.pruned.zst from file /task_172070071020138/artifact0/lex.s2t.pruned.zst with content encoding "identity", mime type "application/zstd" and expiry 2024-10-07T19:15:52.481Z
[taskcluster 2024-07-11T12:29:07.550Z] [mounts] Preserving cache: Moving "/task_172070071020138/cache0" to "caches/fPw0otwNT9OCcyJAGx7wag"
[taskcluster 2024-07-11T12:29:07.583Z] [mounts] Denying task_172070071020138 access to 'caches/fPw0otwNT9OCcyJAGx7wag'
[taskcluster 2024-07-11T12:29:07.652Z] Uploading link artifact public/logs/live.log to artifact public/logs/live_backing.log with expiry 2024-10-07T19:15:52.481Z