[taskcluster 2024-10-22T18:52:09.259Z] Worker Type (translations-1/b-linux-large-gcp-1tb-32-256-d2g) settings:
[taskcluster 2024-10-22T18:52:09.259Z]   {
[taskcluster 2024-10-22T18:52:09.259Z]     "config": {
[taskcluster 2024-10-22T18:52:09.259Z]       "deploymentId": ""
[taskcluster 2024-10-22T18:52:09.259Z]     },
[taskcluster 2024-10-22T18:52:09.259Z]     "generic-worker": {
[taskcluster 2024-10-22T18:52:09.259Z]       "engine": "multiuser",
[taskcluster 2024-10-22T18:52:09.259Z]       "go-arch": "amd64",
[taskcluster 2024-10-22T18:52:09.259Z]       "go-os": "linux",
[taskcluster 2024-10-22T18:52:09.259Z]       "go-version": "go1.22.2",
[taskcluster 2024-10-22T18:52:09.259Z]       "release": "https://github.com/taskcluster/taskcluster/releases/tag/v64.3.0",
[taskcluster 2024-10-22T18:52:09.259Z]       "revision": "b66b6614b9b79077f1bfc78f97afb8c25c0c4b30",
[taskcluster 2024-10-22T18:52:09.259Z]       "source": "https://github.com/taskcluster/taskcluster/commits/b66b6614b9b79077f1bfc78f97afb8c25c0c4b30",
[taskcluster 2024-10-22T18:52:09.259Z]       "version": "64.3.0"
[taskcluster 2024-10-22T18:52:09.259Z]     },
[taskcluster 2024-10-22T18:52:09.259Z]     "image": "projects/taskcluster-imaging/global/images/gw-fxci-gcp-l1-gui-googlecompute-2024-08-22t22-48-09z",
[taskcluster 2024-10-22T18:52:09.259Z]     "instance-id": "1334929993955430197",
[taskcluster 2024-10-22T18:52:09.259Z]     "instance-type": "projects/887720501152/machineTypes/n2-highmem-32",
[taskcluster 2024-10-22T18:52:09.259Z]     "local-ipv4": "10.138.1.5",
[taskcluster 2024-10-22T18:52:09.259Z]     "project-id": "fxci-production-level1-workers",

...(86 lines hidden)...

[taskcluster:warn 2024-10-22T18:52:09.263Z]         taskId: WhHlxCxnQYW05fB1pTTteQ
[taskcluster:warn 2024-10-22T18:52:09.263Z]       file: dockerimage
[taskcluster:warn 2024-10-22T18:52:09.263Z]       format: zst
[taskcluster:warn 2024-10-22T18:52:09.263Z]     onExitStatus:
[taskcluster:warn 2024-10-22T18:52:09.263Z]       purgeCaches:
[taskcluster:warn 2024-10-22T18:52:09.263Z]       - 72
[taskcluster:warn 2024-10-22T18:52:09.263Z]       retry:
[taskcluster:warn 2024-10-22T18:52:09.263Z]       - 128
[taskcluster:warn 2024-10-22T18:52:09.263Z]       - 72
[taskcluster:warn 2024-10-22T18:52:09.263Z]       - 125
[taskcluster:warn 2024-10-22T18:52:09.263Z]   priority: low
[taskcluster:warn 2024-10-22T18:52:09.263Z]   projectId: none
[taskcluster:warn 2024-10-22T18:52:09.263Z]   provisionerId: translations-1
[taskcluster:warn 2024-10-22T18:52:09.263Z]   requires: all-completed
[taskcluster:warn 2024-10-22T18:52:09.263Z]   retries: 5
[taskcluster:warn 2024-10-22T18:52:09.263Z]   routes:
[taskcluster:warn 2024-10-22T18:52:09.263Z]   - index.translations.cache.pr.cefilter.cefilter-ru-en.hash.ab711ea041c13a792602e89f5d3079d6dbebf95486fc7365e78bb6382e86e477
[taskcluster:warn 2024-10-22T18:52:09.263Z]   - checks
[taskcluster:warn 2024-10-22T18:52:09.263Z]   schedulerId: translations-level-1
[taskcluster:warn 2024-10-22T18:52:09.263Z]   scopes:
[taskcluster:warn 2024-10-22T18:52:09.263Z]   - generic-worker:cache:translations-level-1-checkouts-v3-7afeb851dd97df8f3607-WhHlxCxnQYW05fB1pTTteQ
[taskcluster:warn 2024-10-22T18:52:09.263Z]   tags:
[taskcluster:warn 2024-10-22T18:52:09.263Z]     createdForUser: [email protected]
[taskcluster:warn 2024-10-22T18:52:09.263Z]     kind: cefilter
[taskcluster:warn 2024-10-22T18:52:09.263Z]     label: cefilter-ru-en
[taskcluster:warn 2024-10-22T18:52:09.263Z]     os: linux
[taskcluster:warn 2024-10-22T18:52:09.263Z]     worker-implementation: docker-worker
[taskcluster:warn 2024-10-22T18:52:09.263Z]   taskGroupId: M7Sdofq1Tf2gw4K9gkgejA
[taskcluster:warn 2024-10-22T18:52:09.263Z]   taskQueueId: translations-1/b-linux-large-gcp-1tb-32-256-d2g
[taskcluster:warn 2024-10-22T18:52:09.263Z]   workerType: b-linux-large-gcp-1tb-32-256-d2g
[taskcluster:warn 2024-10-22T18:52:09.263Z] 
[taskcluster 2024-10-22T18:52:10.008Z] Uploading redirect artifact public/logs/live.log to URL https://firefoxci-websocktunnel.services.mozilla.com/us-west1-a.1334929993955430197.60099/log/LZxtGAomRSelYHG1I0mnhQ with mime type "text/plain; charset=utf-8" and expiry 2024-10-29T19:22:09.981Z
[taskcluster 2024-10-22T18:52:10.344Z] [mounts] No existing writable directory cache 'translations-level-1-checkouts-v3-7afeb851dd97df8f3607-WhHlxCxnQYW05fB1pTTteQ' - creating caches/NbUJtQxfQJaq88V2xlrX9g
[taskcluster 2024-10-22T18:52:10.344Z] [mounts] Creating directory /task_172962273558407/cache0
[taskcluster 2024-10-22T18:52:10.352Z] [mounts] Granting task_172962273558407 full control of directory '/task_172962273558407/cache0'
[taskcluster 2024-10-22T18:52:10.353Z] [mounts] Successfully mounted writable directory cache '/task_172962273558407/cache0'
[taskcluster:warn 2024-10-22T18:52:13.549Z] [mounts] No SHA256 specified in task mounts for artifact:WhHlxCxnQYW05fB1pTTteQ:public/image.tar.zst - SHA256 from downloaded file downloads/dovdBr3cRsGSZJby3C3CpA is 3acab4e70c157d9e7d6337642cc370f865c7c8eb4056bec81c456bc6caeb0906.
[taskcluster 2024-10-22T18:52:13.549Z] [mounts] Creating directory /task_172962273558407
[taskcluster 2024-10-22T18:52:13.558Z] [mounts] Decompressing zst file downloads/dovdBr3cRsGSZJby3C3CpA to '/task_172962273558407/dockerimage'
[taskcluster 2024-10-22T18:52:14.679Z] [mounts] Granting task_172962273558407 full control of file '/task_172962273558407/dockerimage'
[taskcluster 2024-10-22T18:52:14.680Z] Executing command 0: bash -cx 'timeout 604800 podman run -t --name taskcontainer --privileged -v "$(pwd)/cache0:/builds/worker/checkouts" --add-host=taskcluster:127.0.0.1 --net=host -e FIREFOX_TRANSLATIONS_TRAINING_BASE_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REF -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REV -e FIREFOX_TRANSLATIONS_TRAINING_REPOSITORY_TYPE -e HG_STORE_PATH -e MOZ_AUTOMATION -e MOZ_FETCHES -e MOZ_FETCHES_DIR -e MOZ_SCM_LEVEL -e REPOSITORIES -e RUN_ID -e SCCACHE_DISABLE -e SRC -e TASKCLUSTER_CACHES -e TASKCLUSTER_PROXY_URL -e TASKCLUSTER_ROOT_URL -e TASKCLUSTER_UNTRUSTED_CACHES -e TASKCLUSTER_VOLUMES -e TASKCLUSTER_WORKER_LOCATION -e TASK_ID -e TRG -e VCS_PATH docker-archive:dockerimage /usr/local/bin/run-task --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c '\''$VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus $TASK_WORKDIR/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt'\''
[taskcluster 2024-10-22T18:52:14.680Z] exit_code=$?
[taskcluster 2024-10-22T18:52:14.680Z] podman cp taskcontainer:/builds/worker/artifacts artifact0
[taskcluster 2024-10-22T18:52:14.680Z] podman rm taskcontainer
[taskcluster 2024-10-22T18:52:14.680Z] exit "${exit_code}"'
++ pwd
+ timeout 604800 podman run -t --name taskcontainer --privileged -v /task_172962273558407/cache0:/builds/worker/checkouts --add-host=taskcluster:127.0.0.1 --net=host -e FIREFOX_TRANSLATIONS_TRAINING_BASE_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REF -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REPOSITORY -e FIREFOX_TRANSLATIONS_TRAINING_HEAD_REV -e FIREFOX_TRANSLATIONS_TRAINING_REPOSITORY_TYPE -e HG_STORE_PATH -e MOZ_AUTOMATION -e MOZ_FETCHES -e MOZ_FETCHES_DIR -e MOZ_SCM_LEVEL -e REPOSITORIES -e RUN_ID -e SCCACHE_DISABLE -e SRC -e TASKCLUSTER_CACHES -e TASKCLUSTER_PROXY_URL -e TASKCLUSTER_ROOT_URL -e TASKCLUSTER_UNTRUSTED_CACHES -e TASKCLUSTER_VOLUMES -e TASKCLUSTER_WORKER_LOCATION -e TASK_ID -e TRG -e VCS_PATH docker-archive:dockerimage /usr/local/bin/run-task --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c '$VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus $TASK_WORKDIR/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt'
Getting image source signatures
Copying blob sha256:60b2bd7c53dd80b6b19c7a430a64dad7e0067a2e981484c278b8dfa5b8c93a34
Copying blob sha256:2573e0d8158209ed54ab25c87bcdcb00bd3d2539246960a3d592a1c599d70465
Copying blob sha256:4d1b371a4fd817d79d3765f4ae69b2dc489c12fce54f244a436f8c29c7967f45
Copying config sha256:502a53cc11fc8a59c0ca778225c5a4e38adf93683b8d4000d345029404ff6cf9
Writing manifest to image destination
Storing signatures
[setup 2024-10-22T18:52:28.957Z] run-task started in /builds/worker
[setup 2024-10-22T18:52:28.957Z] Invoked by command: --firefox_translations_training-checkout=/builds/worker/checkouts/vcs/ -- bash -c $VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus $TASK_WORKDIR/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt
[setup 2024-10-22T18:52:28.957Z] Python version: 3.10.12
[cache 2024-10-22T18:52:28.958Z] cache /builds/worker/checkouts is empty; writing requirements: gid=1000 uid=1000 version=1
[volume 2024-10-22T18:52:28.959Z] volume /builds/worker/checkouts is a cache
[setup 2024-10-22T18:52:28.959Z] running as worker:worker
[vcs 2024-10-22T18:52:28.959Z] executing ['git', 'config', '--global', '--add', 'safe.directory', '/builds/worker/checkouts/vcs']
[vcs 2024-10-22T18:52:28.960Z] executing ['git', 'clone', 'https://github.com/mozilla/firefox-translations-training', '/builds/worker/checkouts/vcs']
[vcs 2024-10-22T18:52:28.962Z] Cloning into '/builds/worker/checkouts/vcs'...
[vcs 2024-10-22T18:52:30.552Z] executing ['git', 'fetch', '--tags', '--force', 'https://github.com/mozilla/firefox-translations-training', 'fix_tc_train_args']
[vcs 2024-10-22T18:52:30.850Z] From https://github.com/mozilla/firefox-translations-training
[vcs 2024-10-22T18:52:30.850Z]  * branch            fix_tc_train_args -> FETCH_HEAD
[vcs 2024-10-22T18:52:30.857Z] executing ['git', 'fetch', '--no-tags', 'https://github.com/mozilla/firefox-translations-training', 'fix_tc_train_args']
[vcs 2024-10-22T18:52:31.146Z] From https://github.com/mozilla/firefox-translations-training
[vcs 2024-10-22T18:52:31.146Z]  * branch            fix_tc_train_args -> FETCH_HEAD
[vcs 2024-10-22T18:52:31.153Z] executing ['git', 'checkout', '-f', '-B', 'fix_tc_train_args', '2d0fb8cf64a76c1014ef9eeac29bbe483d5f3b6c']
[vcs 2024-10-22T18:52:31.251Z] Switched to a new branch 'fix_tc_train_args'
[vcs 2024-10-22T18:52:31.251Z] executing ['git', 'submodule', 'init']
[vcs 2024-10-22T18:52:31.272Z] Submodule '3rd_party/browsermt-marian-dev' (https://github.com/browsermt/marian-dev) registered for path '3rd_party/browsermt-marian-dev'
[vcs 2024-10-22T18:52:31.273Z] Submodule 'extract-lex' (https://github.com/marian-nmt/extract-lex) registered for path '3rd_party/extract-lex'
[vcs 2024-10-22T18:52:31.274Z] Submodule 'fast_align' (https://github.com/clab/fast_align) registered for path '3rd_party/fast_align'
[vcs 2024-10-22T18:52:31.275Z] Submodule '3rd_party/kenlm' (https://github.com/kpu/kenlm) registered for path '3rd_party/kenlm'
[vcs 2024-10-22T18:52:31.275Z] Submodule '3rd_party/marian-dev' (https://github.com/marian-nmt/marian-dev) registered for path '3rd_party/marian-dev'
[vcs 2024-10-22T18:52:31.276Z] Submodule '3rd_party/preprocess' (https://github.com/kpu/preprocess.git) registered for path '3rd_party/preprocess'
[vcs 2024-10-22T18:52:31.277Z] Submodule 'inference/3rd_party/browsermt-marian-dev' (https://github.com/browsermt/marian-dev) registered for path 'inference/3rd_party/browsermt-marian-dev'
[vcs 2024-10-22T18:52:31.278Z] Submodule 'inference/3rd_party/ssplit-cpp' (https://github.com/browsermt/ssplit-cpp) registered for path 'inference/3rd_party/ssplit-cpp'
[vcs 2024-10-22T18:52:31.278Z] executing ['git', 'submodule', 'update', '--force']
[vcs 2024-10-22T18:52:31.300Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/browsermt-marian-dev'...
[vcs 2024-10-22T18:52:32.376Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/extract-lex'...
[vcs 2024-10-22T18:52:32.751Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/fast_align'...
[vcs 2024-10-22T18:52:33.212Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/kenlm'...
[vcs 2024-10-22T18:52:34.144Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/marian-dev'...
[vcs 2024-10-22T18:52:35.979Z] Cloning into '/builds/worker/checkouts/vcs/3rd_party/preprocess'...
[vcs 2024-10-22T18:52:36.839Z] Cloning into '/builds/worker/checkouts/vcs/inference/3rd_party/browsermt-marian-dev'...
[vcs 2024-10-22T18:52:37.928Z] Cloning into '/builds/worker/checkouts/vcs/inference/3rd_party/ssplit-cpp'...
[vcs 2024-10-22T18:52:38.412Z] Submodule path '3rd_party/browsermt-marian-dev': checked out '11c6ae7c46be21ef96ed10c60f28022fa968939f'
[vcs 2024-10-22T18:52:38.424Z] Submodule path '3rd_party/extract-lex': checked out '42fa605b53f32eaf6c6e0b5677255c21c91b3d49'
[vcs 2024-10-22T18:52:38.437Z] Submodule path '3rd_party/fast_align': checked out 'cab1e9aac8d3bb02ff5ae58218d8d225a039fa11'
[vcs 2024-10-22T18:52:38.465Z] Submodule path '3rd_party/kenlm': checked out 'bbf4fc511266c5d4515047055d7bdec659a6e158'
[vcs 2024-10-22T18:52:38.566Z] Submodule path '3rd_party/marian-dev': checked out 'e8a1a2530fb84cbff7383302ebca393e5875c441'
[vcs 2024-10-22T18:52:38.586Z] Submodule path '3rd_party/preprocess': checked out '64307314b4d5a9a0bd529b5c1036b0710d995eec'
[vcs 2024-10-22T18:52:38.649Z] Submodule path 'inference/3rd_party/browsermt-marian-dev': checked out '2781d735d4a10dca876d61be587afdab2726293c'
[vcs 2024-10-22T18:52:38.663Z] Submodule path 'inference/3rd_party/ssplit-cpp': checked out 'a311f9865ade34db1e8e080e6cc146f55dafb067'
[vcs 2024-10-22T18:52:38.664Z] cleaning git checkout...
[vcs 2024-10-22T18:52:38.664Z] executing ['git', 'clean', '-nxdff']
[vcs 2024-10-22T18:52:38.667Z] removing []
[vcs 2024-10-22T18:52:38.667Z] successfully cleaned git checkout!
[vcs 2024-10-22T18:52:38.668Z] TinderboxPrint:<a href='https://github.com/mozilla/firefox-translations-training/commit/2d0fb8cf64a76c1014ef9eeac29bbe483d5f3b6c' title='Built from firefox-translations-training commit 2d0fb8cf64a76c1014ef9eeac29bbe483d5f3b6c'>2d0fb8cf64a76c1014ef9eeac29bbe483d5f3b6c</a>
[setup 2024-10-22T18:52:38.668Z] MOZ_FETCHES_DIR is /builds/worker/fetches
[fetches 2024-10-22T18:52:38.668Z] fetching artifacts
[fetches 2024-10-22T18:52:38.668Z] executing ['/usr/bin/python3', '-u', '/usr/local/bin/fetch-content', 'task-artifacts']
attempt 1/5
attempt 1/5Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AP1T9tqSQ4WV4v23gm-97Q/artifacts/public/build/corpus.en.zst to /builds/worker/fetches/corpus.en.zst
attempt 1/5

Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AP1T9tqSQ4WV4v23gm-97Q/artifacts/public/build/corpus.ru.zst to /builds/worker/fetches/corpus.ru.zstDownloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AP1T9tqSQ4WV4v23gm-97Q/artifacts/public/build/corpus.en.zstDownloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AhGYgi-DRcqGOd_3WLQRPA/artifacts/public/build/scores.txt to /builds/worker/fetches/scores.txt


Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AP1T9tqSQ4WV4v23gm-97Q/artifacts/public/build/corpus.ru.zst
Downloading https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AhGYgi-DRcqGOd_3WLQRPA/artifacts/public/build/scores.txt
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AP1T9tqSQ4WV4v23gm-97Q/artifacts/public/build/corpus.en.zst resolved to 8710 bytes with sha256 aa6a0f45a338d39d6802591a3b93547dfd723709af89ca90a7f60abf6159a71a in 0.096s
Verified size of https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AP1T9tqSQ4WV4v23gm-97Q/artifacts/public/build/corpus.en.zst
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AP1T9tqSQ4WV4v23gm-97Q/artifacts/public/build/corpus.ru.zst resolved to 88320 bytes with sha256 6f4ab09eb1b2e3ecdfa74d6e84a40237abe8590d74d82a6fd46ac7f31976fc3f in 0.121s
Verified size of https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AP1T9tqSQ4WV4v23gm-97Q/artifacts/public/build/corpus.ru.zst
https://firefox-ci-tc.services.mozilla.com/api/queue/v1/task/AhGYgi-DRcqGOd_3WLQRPA/artifacts/public/build/scores.txt resolved to 16920 bytes with sha256 1f54c832b35d5c20e26ac43975b6730330a0fc4e96aa9ea68a95583e99157c5b in 0.164s
Extracting /builds/worker/fetches/scores.txt to /builds/worker/fetches
PERFHERDER_DATA: {"framework": {"name": "build_metrics"}, "suites": [{"name": "fetch_content", "value": 0.1666976240000082, "lowerIsBetter": true, "shouldAlert": false, "subtests": []}]}
[fetches 2024-10-22T18:52:38.903Z] finished fetching artifacts
[task 2024-10-22T18:52:38.903Z] executing ['bash', '-c', '$VCS_PATH/pipeline/cefilter/ce-filter.sh $MOZ_FETCHES_DIR/corpus $TASK_WORKDIR/artifacts/corpus $MOZ_FETCHES_DIR/scores.txt']
[task 2024-10-22T18:52:38.905Z] + set -euo pipefail
[task 2024-10-22T18:52:38.905Z] + echo '###### Cross entropy filtering'
[task 2024-10-22T18:52:38.905Z] ###### Cross entropy filtering
[task 2024-10-22T18:52:38.905Z] + test -v SRC
[task 2024-10-22T18:52:38.905Z] + test -v TRG
[task 2024-10-22T18:52:38.905Z] + corpus_prefix=/builds/worker/fetches/corpus
[task 2024-10-22T18:52:38.905Z] + output_prefix=/builds/worker/artifacts/corpus
[task 2024-10-22T18:52:38.905Z] + scores=/builds/worker/fetches/scores.txt
[task 2024-10-22T18:52:38.905Z] ++ dirname /builds/worker/checkouts/vcs/pipeline/cefilter/ce-filter.sh
[task 2024-10-22T18:52:38.906Z] + cd /builds/worker/checkouts/vcs/pipeline/cefilter
[task 2024-10-22T18:52:38.906Z] + remove=0.05
[task 2024-10-22T18:52:38.906Z] ++ dirname /builds/worker/artifacts/corpus
[task 2024-10-22T18:52:38.907Z] + output_dir=/builds/worker/artifacts
[task 2024-10-22T18:52:38.907Z] + tmp=/builds/worker/artifacts/tmp
[task 2024-10-22T18:52:38.907Z] + mkdir -p /builds/worker/artifacts/tmp
[task 2024-10-22T18:52:38.935Z] + echo '### Sorting scores'
[task 2024-10-22T18:52:38.935Z] ### Sorting scores
[task 2024-10-22T18:52:38.935Z] + '[' '!' -s /builds/worker/artifacts/tmp/sorted.zst ']'
[task 2024-10-22T18:52:38.936Z] ++ bc
[task 2024-10-22T18:52:38.936Z] ++ cut -f1 -d.
[task 2024-10-22T18:52:38.936Z] +++ grep MemTotal /proc/meminfo
[task 2024-10-22T18:52:38.936Z] +++ awk '{print $2}'
[task 2024-10-22T18:52:38.938Z] ++ echo '264108036*0.9'
[task 2024-10-22T18:52:38.938Z] + buffer_size=237697232
[task 2024-10-22T18:52:38.939Z] + LC_ALL=C
[task 2024-10-22T18:52:38.939Z] + sort -n -k1,1 -S 237697232K -T /builds/worker/artifacts/tmp
[task 2024-10-22T18:52:38.939Z] + zstdmt
[task 2024-10-22T18:52:38.939Z] ++ zstdmt -dc /builds/worker/fetches/corpus.ru.zst
[task 2024-10-22T18:52:38.939Z] + paste /builds/worker/fetches/scores.txt /dev/fd/63 /dev/fd/62
[task 2024-10-22T18:52:38.939Z] ++ zstdmt -dc /builds/worker/fetches/corpus.en.zst
[task 2024-10-22T18:52:38.948Z] + echo '### Cutting the best scored corpus'
[task 2024-10-22T18:52:38.948Z] ### Cutting the best scored corpus
[task 2024-10-22T18:52:38.948Z] + '[' '!' -s /builds/worker/artifacts/tmp/best.zst ']'
[task 2024-10-22T18:52:38.949Z] ++ zstdmt -dc /builds/worker/artifacts/tmp/sorted.zst
[task 2024-10-22T18:52:38.949Z] ++ wc -l
[task 2024-10-22T18:52:38.951Z] + lines=1692
[task 2024-10-22T18:52:38.952Z] ++ echo '1692*0.05'
[task 2024-10-22T18:52:38.952Z] ++ bc
[task 2024-10-22T18:52:38.952Z] ++ cut -f1 -d.
[task 2024-10-22T18:52:38.953Z] + startline=84
[task 2024-10-22T18:52:38.953Z] + zstdmt -dc /builds/worker/artifacts/tmp/sorted.zst
[task 2024-10-22T18:52:38.953Z] + tail -n +84
[task 2024-10-22T18:52:38.953Z] + cut -f2,3
[task 2024-10-22T18:52:38.953Z] + zstdmt
[task 2024-10-22T18:52:38.960Z] + echo '### Writing output corpus'
[task 2024-10-22T18:52:38.960Z] ### Writing output corpus
[task 2024-10-22T18:52:38.960Z] + zstdmt -dc /builds/worker/artifacts/tmp/best.zst
[task 2024-10-22T18:52:38.960Z] + cut -f2
[task 2024-10-22T18:52:38.960Z] + tee /dev/fd/63
[task 2024-10-22T18:52:38.960Z] + zstdmt
[task 2024-10-22T18:52:38.961Z] ++ cut -f1
[task 2024-10-22T18:52:38.961Z] ++ zstdmt
[task 2024-10-22T18:52:38.968Z] + echo '### Deleting tmp dir'
[task 2024-10-22T18:52:38.969Z] ### Deleting tmp dir
[task 2024-10-22T18:52:38.969Z] + rm -rf /builds/worker/artifacts/tmp
[task 2024-10-22T18:52:38.969Z] + echo '###### Done: Cross entropy filtering'
[task 2024-10-22T18:52:38.969Z] ###### Done: Cross entropy filtering
[fetches 2024-10-22T18:52:38.970Z] removing /builds/worker/fetches
[fetches 2024-10-22T18:52:38.970Z] finished
+ exit_code=0
+ podman cp taskcontainer:/builds/worker/artifacts artifact0
+ podman rm taskcontainer
976bcdcd47e73067b7fbe9750799617af1d5975efaabc58a52b82ad132bc4fb9
+ exit 0
[taskcluster 2024-10-22T18:52:41.885Z]    Exit Code: 0
[taskcluster 2024-10-22T18:52:41.885Z]    User Time: 9.041265s
[taskcluster 2024-10-22T18:52:41.885Z]  Kernel Time: 3.660244s
[taskcluster 2024-10-22T18:52:41.885Z]    Wall Time: 27.204187699s
[taskcluster 2024-10-22T18:52:41.885Z]       Result: SUCCEEDED
[taskcluster 2024-10-22T18:52:41.885Z] === Task Finished ===
[taskcluster 2024-10-22T18:52:41.885Z] Task Duration: 27.204482835s
[taskcluster 2024-10-22T18:52:41.920Z] Uploading artifact public/build/corpus.ru.zst from file /task_172962273558407/artifact0/corpus.ru.zst with content encoding "identity", mime type "application/zstd" and expiry 2025-10-17T18:00:07.912Z
[taskcluster 2024-10-22T18:52:41.922Z] Uploading artifact public/build/corpus.en.zst from file /task_172962273558407/artifact0/corpus.en.zst with content encoding "identity", mime type "application/zstd" and expiry 2025-10-17T18:00:07.912Z
[taskcluster 2024-10-22T18:52:42.080Z] [mounts] Preserving cache: Moving "/task_172962273558407/cache0" to "caches/NbUJtQxfQJaq88V2xlrX9g"
[taskcluster 2024-10-22T18:52:42.080Z] [mounts] Denying task_172962273558407 access to 'caches/NbUJtQxfQJaq88V2xlrX9g'
[taskcluster 2024-10-22T18:52:42.104Z] Uploading link artifact public/logs/live.log to artifact public/logs/live_backing.log with expiry 2025-10-17T18:00:07.912Z