[taskcluster 2024-10-22T19:07:06.685Z] Worker Type (translations-1/b-linux-v100-gpu-4-300gb) settings:
[taskcluster 2024-10-22T19:07:06.685Z]   {
[taskcluster 2024-10-22T19:07:06.685Z]     "config": {
[taskcluster 2024-10-22T19:07:06.685Z]       "deploymentId": ""
[taskcluster 2024-10-22T19:07:06.685Z]     },
[taskcluster 2024-10-22T19:07:06.685Z]     "generic-worker": {
[taskcluster 2024-10-22T19:07:06.685Z]       "engine": "insecure",
[taskcluster 2024-10-22T19:07:06.685Z]       "go-arch": "amd64",
[taskcluster 2024-10-22T19:07:06.685Z]       "go-os": "linux",
[taskcluster 2024-10-22T19:07:06.685Z]       "go-version": "go1.22.2",
[taskcluster 2024-10-22T19:07:06.685Z]       "release": "https://github.com/taskcluster/taskcluster/releases/tag/v64.2.6",
[taskcluster 2024-10-22T19:07:06.685Z]       "revision": "edab196d7d030a5d625b77335109cd9060ab7e1f",
[taskcluster 2024-10-22T19:07:06.685Z]       "source": "https://github.com/taskcluster/taskcluster/commits/edab196d7d030a5d625b77335109cd9060ab7e1f",
[taskcluster 2024-10-22T19:07:06.685Z]       "version": "64.2.6"
[taskcluster 2024-10-22T19:07:06.685Z]     },
[taskcluster 2024-10-22T19:07:06.685Z]     "image": "projects/taskcluster-imaging/global/images/gw-translations-gcp-googlecompute-2024-04-22t18-22-42z",
[taskcluster 2024-10-22T19:07:06.685Z]     "instance-id": "3663911460871489625",
[taskcluster 2024-10-22T19:07:06.685Z]     "instance-type": "projects/887720501152/machineTypes/custom-40-262144",
[taskcluster 2024-10-22T19:07:06.685Z]     "local-ipv4": "10.142.0.47",
[taskcluster 2024-10-22T19:07:06.685Z]     "project-id": "fxci-production-level1-workers",

...(6114 lines hidden)...

[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1320
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1321
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1322
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1323
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1324
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1325
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1326
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1327
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1328
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1329
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1330
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1331
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1332
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1333
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1334
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1335
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1336
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1337
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1338
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1339
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1340
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1341
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1342
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1343
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1344
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1345
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1346
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1347
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1348
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1349
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1350
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1351
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1352
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1353
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1354
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1355
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1356
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1357
[task 2024-10-22T19:15:39.386Z] 
[task 2024-10-22T19:15:39.386Z] [tag] Reading original for epoch 1358
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1359
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1360
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1361
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1362
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1363
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1364
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1365
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1366
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1367
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1368
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1369
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1370
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1371
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1372
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1373
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1374
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1375
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1376
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1377
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1378
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1379
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1380
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1381
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:39.387Z] [tag] Reading original for epoch 1382
[task 2024-10-22T19:15:39.387Z] 
[task 2024-10-22T19:15:42.561Z] wandb: - 0.006 MB of 0.006 MB uploaded
[task 2024-10-22T19:15:43.414Z] wandb: \ 0.006 MB of 0.124 MB uploaded
[task 2024-10-22T19:15:43.416Z] wandb: | 0.006 MB of 0.124 MB uploaded
[task 2024-10-22T19:15:43.416Z] wandb: / 0.124 MB of 0.124 MB uploaded
[task 2024-10-22T19:15:43.416Z] wandb:                                                                                
[task 2024-10-22T19:15:44.421Z] wandb: 
[task 2024-10-22T19:15:44.421Z] wandb: Run history:
[task 2024-10-22T19:15:44.421Z] wandb:    bleu_detok ▁
[task 2024-10-22T19:15:44.421Z] wandb: ce_mean_words ▁
[task 2024-10-22T19:15:44.421Z] wandb:          chrf ▁
[task 2024-10-22T19:15:44.421Z] wandb:          cost ▆▇▃▂▇▂▁▂▁▂▃█▇▇█▁█▂▃▁▂▂▇██▂▇▃█▁▂▇▄▇▆▂▄▇▆▇
[task 2024-10-22T19:15:44.421Z] wandb:         epoch ▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
[task 2024-10-22T19:15:44.421Z] wandb:         gnorm ▃▁▂▂▁▂▃▂▆▅▅▆▇▆▇▇▇▇▇▆▇▇▇██▇▇▇█▇▇█▇▇▇▆▆▆▅▅
[task 2024-10-22T19:15:44.421Z] wandb: learning_rate ▁▁▁▁▂▂▂▂▂▃▃▃▃▃▃▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▆▇▇▇▇▇███
[task 2024-10-22T19:15:44.421Z] wandb:          rate ▁▂▃▅▁▃▅▄▇▅▄▁▁▁▁▇▁▇▅█▁▅▁▁▁▃▁▂▁▃▆▁▂▁▁▅▂▂▁▁
[task 2024-10-22T19:15:44.421Z] wandb:           sen ▁▁▁▁▂▂▂▂▂▂▂▃▃▃▃▃▄▄▄▄▄▄▄▄▅▅▅▅▅▅▆▆▆▆▆▇▇▇▇█
[task 2024-10-22T19:15:44.421Z] wandb:          time █▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁▁
[task 2024-10-22T19:15:44.421Z] wandb: 
[task 2024-10-22T19:15:44.421Z] wandb: Run summary:
[task 2024-10-22T19:15:44.421Z] wandb:    bleu_detok 0.0
[task 2024-10-22T19:15:44.421Z] wandb: ce_mean_words 7.62995
[task 2024-10-22T19:15:44.421Z] wandb:          chrf 0.9002
[task 2024-10-22T19:15:44.421Z] wandb:          cost 6.34634
[task 2024-10-22T19:15:44.421Z] wandb:         epoch 1
[task 2024-10-22T19:15:44.421Z] wandb:         gnorm 44.194
[task 2024-10-22T19:15:44.421Z] wandb: learning_rate 0.0
[task 2024-10-22T19:15:44.421Z] wandb:          rate 67024.54
[task 2024-10-22T19:15:44.421Z] wandb:           sen 153574
[task 2024-10-22T19:15:44.421Z] wandb:          time 0.27
[task 2024-10-22T19:15:44.421Z] wandb: 
[task 2024-10-22T19:15:44.425Z] wandb: 🚀 View run student_M7Sdo at: https://wandb.ai/moz-translations/ci/runs/student_M7Sdo
[task 2024-10-22T19:15:44.425Z] wandb: ⭐️ View project at: https://wandb.ai/moz-translations/ci
[task 2024-10-22T19:15:44.425Z] wandb: Synced 5 W&B file(s), 1 media file(s), 1 artifact file(s) and 0 other file(s)
[task 2024-10-22T19:15:44.425Z] wandb: Find logs at: ./wandb/run-20241022_190846-student_M7Sdo/logs
[task 2024-10-22T19:15:44.425Z] wandb: WARNING The new W&B backend becomes opt-out in version 0.18.0; try it out with `wandb.require("core")`! See https://wandb.me/wandb-core for more information.
[task 2024-10-22T19:15:44.464Z] [tracking INFO] Successfully parsed 1922 lines
[task 2024-10-22T19:15:44.464Z] [tracking INFO] Found 50 training entries
[task 2024-10-22T19:15:44.464Z] [tracking INFO] Found 1 validation entries
[task 2024-10-22T19:15:44.464Z] [tag] Reading original for epoch 1383
[task 2024-10-22T19:15:44.464Z] 
[task 2024-10-22T19:15:44.464Z] [tag] trainer stopped reading input
[task 2024-10-22T19:15:44.464Z] 
[fetches 2024-10-22T19:15:46.703Z] removing /home/ubuntu/tasks/task_172962398799594/fetches
[fetches 2024-10-22T19:15:47.063Z] finished
[taskcluster 2024-10-22T19:15:47.075Z]    Exit Code: 0
[taskcluster 2024-10-22T19:15:47.075Z]    User Time: 30m11.385277s
[taskcluster 2024-10-22T19:15:47.075Z]  Kernel Time: 1m33.741445s
[taskcluster 2024-10-22T19:15:47.075Z]    Wall Time: 8m38.839700605s
[taskcluster 2024-10-22T19:15:47.075Z]       Result: SUCCEEDED
[taskcluster 2024-10-22T19:15:47.075Z] === Task Finished ===
[taskcluster 2024-10-22T19:15:47.075Z] Task Duration: 8m38.841750393s
[taskcluster 2024-10-22T19:15:47.331Z] Uploading artifact public/build/config.opustrainer.yml.state from file /home/ubuntu/tasks/task_172962398799594/artifacts/config.opustrainer.yml.state with content encoding "gzip", mime type "application/octet-stream" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.334Z] Uploading artifact public/build/model.npz.progress.yml from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.progress.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.344Z] Uploading artifact public/build/model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.best-chrf.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.346Z] Uploading artifact public/build/final.model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_172962398799594/artifacts/final.model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.347Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.best-bleu-detok.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.349Z] Uploading artifact public/build/train.log from file /home/ubuntu/tasks/task_172962398799594/artifacts/train.log with content encoding "gzip", mime type "text/plain" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.349Z] Uploading artifact public/build/devset.out from file /home/ubuntu/tasks/task_172962398799594/artifacts/devset.out with content encoding "gzip", mime type "application/octet-stream" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.351Z] Uploading artifact public/build/final.model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_172962398799594/artifacts/final.model.npz.best-chrf.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.351Z] Uploading artifact public/build/model.npz.optimizer.npz from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.optimizer.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.354Z] Uploading artifact public/build/model.npz.yml from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.355Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz.decoder.yml from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.best-bleu-detok.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.361Z] Uploading artifact public/build/model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.362Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.best-ce-mean-words.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.362Z] Uploading artifact public/build/vocab.spm from file /home/ubuntu/tasks/task_172962398799594/artifacts/vocab.spm with content encoding "gzip", mime type "application/x-source-rpm" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.363Z] Uploading artifact public/build/config.opustrainer.yml from file /home/ubuntu/tasks/task_172962398799594/artifacts/config.opustrainer.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.365Z] Uploading artifact public/build/valid.log from file /home/ubuntu/tasks/task_172962398799594/artifacts/valid.log with content encoding "gzip", mime type "text/plain" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.365Z] Uploading artifact public/build/model.npz from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.366Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz.decoder.yml from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.best-ce-mean-words.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.376Z] Uploading artifact public/build/opustrainer.log from file /home/ubuntu/tasks/task_172962398799594/artifacts/opustrainer.log with content encoding "gzip", mime type "text/plain" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:47.508Z] Uploading artifact public/build/model.npz.decoder.yml from file /home/ubuntu/tasks/task_172962398799594/artifacts/model.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-10-17T18:00:08.076Z
[taskcluster 2024-10-22T19:15:49.035Z] [mounts] Preserving cache: Moving "/home/ubuntu/tasks/task_172962398799594/checkouts" to "/home/ubuntu/caches/Kdnn3qXJRKioqi2dJFVT6g"
[taskcluster 2024-10-22T19:15:49.168Z] Uploading link artifact public/logs/live.log to artifact public/logs/live_backing.log with expiry 2025-10-17T18:00:08.076Z