[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] right-left: false
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] save-freq: 25
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] seed: 1
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sentencepiece-alphas:
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config]   - 0.5
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sentencepiece-max-lines: 2000000
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sentencepiece-options: ""
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sharding: local
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] shuffle: batches
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] shuffle-in-ram: false
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sigterm: save-and-exit
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] skip: false
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sqlite: ""
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sqlite-drop: false
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sync-freq: 200u
[task 2024-07-10T20:40:35.632Z] [2024-07-10 20:40:35] [config] sync-sgd: true
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] tempdir: /home/ubuntu/tasks/task_172064391152108/artifacts/tmp
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] throw-on-divergence:
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config]   []
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] tied-embeddings: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] tied-embeddings-all: true
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] tied-embeddings-src: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] train-embedder-rank:
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config]   []
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] train-sets:
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config]   - stdin
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-aan-activation: swish
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-aan-depth: 2
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-aan-nogate: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-decoder-autoreg: self-attention
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-decoder-dim-ffn: 0
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-decoder-ffn-depth: 0
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-depth-scaling: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-dim-aan: 2048
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-dim-ffn: 2048
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-dropout: 0.1
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-dropout-attention: 0
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-dropout-ffn: 0
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-ffn-activation: relu
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-ffn-depth: 2
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-guided-alignment-layer: last
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-heads: 8
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-no-affine: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-no-bias: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-no-projection: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-pool: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-postprocess: dan
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-postprocess-emb: d
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-postprocess-top: ""
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-preprocess: ""
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-rnn-projection: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-tied-layers:
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config]   []
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] transformer-train-position-embeddings: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] tsv: true
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] tsv-fields: 2
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] type: transformer
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] ulr: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] ulr-dim-emb: 0
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] ulr-dropout: 0
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] ulr-keys-vectors: ""
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] ulr-query-vectors: ""
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] ulr-softmax-temperature: 1
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] ulr-trainable-transformation: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] unlikelihood-loss: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-freq: 50
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-log: /home/ubuntu/tasks/task_172064391152108/artifacts/valid.log
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-max-length: 300
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-metrics:
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config]   - chrf
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config]   - ce-mean-words
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config]   - bleu-detok
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-mini-batch: 16
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-reset-all: false
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-reset-stalled: true
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-script-args:
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config]   []
[task 2024-07-10T20:40:35.633Z] [2024-07-10 20:40:35] [config] valid-script-path: ""
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config] valid-sets:
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config]   - /home/ubuntu/tasks/task_172064391152108/fetches/devset.ruen.tsv
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config] valid-translation-output: /home/ubuntu/tasks/task_172064391152108/artifacts/devset.out
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config] vocabs:
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config]   - /home/ubuntu/tasks/task_172064391152108/artifacts/vocab.spm
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config]   - /home/ubuntu/tasks/task_172064391152108/artifacts/vocab.spm
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config] word-penalty: 0
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config] word-scores: false
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config] workspace: 12000
[task 2024-07-10T20:40:35.634Z] [2024-07-10 20:40:35] [config] Model is being created with Marian v1.12.14 2d067af 2024-02-16 11:44:13 -0500
[task 2024-07-10T20:40:35.635Z] [2024-07-10 20:40:35] Using synchronous SGD
[task 2024-07-10T20:40:35.656Z] [tracking INFO] Detected Marian version 1.12
[task 2024-07-10T20:40:35.656Z] [2024-07-10 20:40:35] [comm] Compiled without MPI support. Running as a single process on translations-1-b-linux-v100-gpu-4-xsatrk8bs9afwpy-vghj6g
[task 2024-07-10T20:40:35.656Z] [2024-07-10 20:40:35] Synced seed 1
[task 2024-07-10T20:40:35.656Z] [2024-07-10 20:40:35] [data] Loading SentencePiece vocabulary from file /home/ubuntu/tasks/task_172064391152108/artifacts/vocab.spm
[task 2024-07-10T20:40:35.656Z] [2024-07-10 20:40:35] [data] Setting vocabulary size for input 0 to 1,000
[task 2024-07-10T20:40:35.656Z] [2024-07-10 20:40:35] [data] Loading SentencePiece vocabulary from file /home/ubuntu/tasks/task_172064391152108/artifacts/vocab.spm
[task 2024-07-10T20:40:35.656Z] [2024-07-10 20:40:35] [data] Setting vocabulary size for input 1 to 1,000
[task 2024-07-10T20:40:35.656Z] [2024-07-10 20:40:35] [batching] Collecting statistics for batch fitting with step size 10
[task 2024-07-10T20:40:36.347Z] [2024-07-10 20:40:36] [memory] Extending reserved space to 12032 MB (device gpu0)
[task 2024-07-10T20:40:36.464Z] [2024-07-10 20:40:36] [memory] Extending reserved space to 12032 MB (device gpu1)
[task 2024-07-10T20:40:36.563Z] [2024-07-10 20:40:36] [memory] Extending reserved space to 12032 MB (device gpu2)
[task 2024-07-10T20:40:36.666Z] [2024-07-10 20:40:36] [memory] Extending reserved space to 12032 MB (device gpu3)
[task 2024-07-10T20:40:36.671Z] [2024-07-10 20:40:36] [comm] Using NCCL 2.8.3 for GPU communication
[task 2024-07-10T20:40:36.671Z] [2024-07-10 20:40:36] [comm] Using global sharding
[task 2024-07-10T20:40:36.944Z] [2024-07-10 20:40:36] [comm] NCCLCommunicators constructed successfully
[task 2024-07-10T20:40:36.944Z] [2024-07-10 20:40:36] [training] Using 4 GPUs
[task 2024-07-10T20:40:36.947Z] [2024-07-10 20:40:36] [logits] Applying loss function for 1 factor(s)
[task 2024-07-10T20:40:36.947Z] [2024-07-10 20:40:36] [memory] Reserving 170 MB, device gpu0
[task 2024-07-10T20:40:37.737Z] [2024-07-10 20:40:37] [gpu] 16-bit TensorCores enabled for float32 matrix operations
[task 2024-07-10T20:40:37.984Z] [2024-07-10 20:40:37] [memory] Reserving 170 MB, device gpu0
[task 2024-07-10T20:40:55.863Z] [2024-07-10 20:40:55] [batching] Done. Typical MB size is 130,616 target words
[task 2024-07-10T20:40:55.958Z] [2024-07-10 20:40:55] [memory] Extending reserved space to 12032 MB (device gpu0)
[task 2024-07-10T20:40:56.248Z] [2024-07-10 20:40:56] [memory] Extending reserved space to 12032 MB (device gpu1)
[task 2024-07-10T20:40:56.264Z] [2024-07-10 20:40:56] [memory] Extending reserved space to 12032 MB (device gpu2)
[task 2024-07-10T20:40:56.288Z] [2024-07-10 20:40:56] [memory] Extending reserved space to 12032 MB (device gpu3)
[task 2024-07-10T20:40:56.307Z] [2024-07-10 20:40:56] [comm] Using NCCL 2.8.3 for GPU communication
[task 2024-07-10T20:40:56.307Z] [2024-07-10 20:40:56] [comm] Using global sharding
[task 2024-07-10T20:40:56.495Z] [2024-07-10 20:40:56] [comm] NCCLCommunicators constructed successfully
[task 2024-07-10T20:40:56.495Z] [2024-07-10 20:40:56] [training] Using 4 GPUs
[task 2024-07-10T20:40:56.495Z] [2024-07-10 20:40:56] Training started
[task 2024-07-10T20:43:48.414Z] [2024-07-10 20:43:48] [training] Batches are processed as 1 process(es) x 4 devices/process
[task 2024-07-10T20:43:48.419Z] [2024-07-10 20:43:48] [memory] Reserving 170 MB, device gpu1
[task 2024-07-10T20:43:48.419Z] [2024-07-10 20:43:48] [memory] Reserving 170 MB, device gpu3
[task 2024-07-10T20:43:48.419Z] [2024-07-10 20:43:48] [memory] Reserving 170 MB, device gpu2
[task 2024-07-10T20:43:48.419Z] [2024-07-10 20:43:48] [memory] Reserving 170 MB, device gpu0
[task 2024-07-10T20:43:48.520Z] [2024-07-10 20:43:48] [memory] Reserving 170 MB, device gpu0
[task 2024-07-10T20:43:48.565Z] [2024-07-10 20:43:48] [memory] Reserving 170 MB, device gpu2
[task 2024-07-10T20:43:48.595Z] [2024-07-10 20:43:48] [memory] Reserving 170 MB, device gpu1
[task 2024-07-10T20:43:48.629Z] [2024-07-10 20:43:48] [memory] Reserving 170 MB, device gpu3
[task 2024-07-10T20:43:49.555Z] [2024-07-10 20:43:49] Parameter type float32, optimization type float32, casting types false
[task 2024-07-10T20:43:49.555Z] [2024-07-10 20:43:49] [memory] Reserving 42 MB, device gpu2
[task 2024-07-10T20:43:49.555Z] [2024-07-10 20:43:49] [memory] Reserving 42 MB, device gpu3
[task 2024-07-10T20:43:49.555Z] [2024-07-10 20:43:49] [memory] Reserving 42 MB, device gpu0
[task 2024-07-10T20:43:49.556Z] [2024-07-10 20:43:49] Allocating memory for general optimizer shards
[task 2024-07-10T20:43:49.556Z] [2024-07-10 20:43:49] [memory] Reserving 42 MB, device gpu1
[task 2024-07-10T20:43:49.564Z] [2024-07-10 20:43:49] Allocating memory for Adam-specific shards
[task 2024-07-10T20:43:49.564Z] [2024-07-10 20:43:49] [memory] Reserving 85 MB, device gpu3
[task 2024-07-10T20:43:49.568Z] [2024-07-10 20:43:49] [memory] Reserving 85 MB, device gpu2
[task 2024-07-10T20:43:49.568Z] [2024-07-10 20:43:49] [memory] Reserving 85 MB, device gpu0
[task 2024-07-10T20:43:49.570Z] [2024-07-10 20:43:49] [memory] Reserving 85 MB, device gpu1
[task 2024-07-10T20:43:49.575Z] [2024-07-10 20:43:49] Ep. 1 : Up. 1 : Sen. 1,336 : Cost 7.33780956 : Time 173.64s : 530.90 words/s : gNorm 7.8010 : L.r. 1.8750e-08
[task 2024-07-10T20:43:50.060Z] [2024-07-10 20:43:50] Ep. 1 : Up. 2 : Sen. 4,704 : Cost 7.34560871 : Time 0.49s : 159588.40 words/s : gNorm 7.6589 : L.r. 3.7500e-08
[task 2024-07-10T20:43:50.387Z] [2024-07-10 20:43:50] Ep. 1 : Up. 3 : Sen. 6,504 : Cost 7.33951569 : Time 0.33s : 115769.56 words/s : gNorm 8.1739 : L.r. 5.6250e-08
[task 2024-07-10T20:43:50.726Z] [2024-07-10 20:43:50] Ep. 1 : Up. 4 : Sen. 8,648 : Cost 7.31468153 : Time 0.34s : 113823.84 words/s : gNorm 8.3939 : L.r. 7.5000e-08
[task 2024-07-10T20:43:51.229Z] [2024-07-10 20:43:51] Ep. 1 : Up. 5 : Sen. 11,280 : Cost 7.30951834 : Time 0.50s : 167511.04 words/s : gNorm 8.1540 : L.r. 9.3750e-08
[task 2024-07-10T20:43:51.734Z] [2024-07-10 20:43:51] Ep. 1 : Up. 6 : Sen. 12,616 : Cost 7.27759266 : Time 0.50s : 158807.54 words/s : gNorm 7.9874 : L.r. 1.1250e-07
[task 2024-07-10T20:43:52.212Z] [2024-07-10 20:43:52] Ep. 1 : Up. 7 : Sen. 15,248 : Cost 7.35211706 : Time 0.48s : 154084.30 words/s : gNorm 7.9525 : L.r. 1.3125e-07
[task 2024-07-10T20:43:52.755Z] [2024-07-10 20:43:52] Ep. 1 : Up. 8 : Sen. 19,848 : Cost 7.34059381 : Time 0.54s : 178035.34 words/s : gNorm 8.0313 : L.r. 1.5000e-07
[task 2024-07-10T20:43:53.369Z] [2024-07-10 20:43:53] Ep. 1 : Up. 9 : Sen. 22,480 : Cost 7.27512455 : Time 0.61s : 188683.67 words/s : gNorm 7.9047 : L.r. 1.6875e-07
[task 2024-07-10T20:43:53.906Z] [2024-07-10 20:43:53] Ep. 1 : Up. 10 : Sen. 23,656 : Cost 7.30049038 : Time 0.54s : 188338.15 words/s : gNorm 7.8177 : L.r. 1.8750e-07
[task 2024-07-10T20:43:54.446Z] [2024-07-10 20:43:54] Ep. 1 : Up. 11 : Sen. 24,832 : Cost 7.22664642 : Time 0.54s : 159114.60 words/s : gNorm 7.7769 : L.r. 2.0625e-07
[task 2024-07-10T20:43:54.842Z] [2024-07-10 20:43:54] Ep. 1 : Up. 12 : Sen. 26,008 : Cost 7.26353216 : Time 0.40s : 133527.28 words/s : gNorm 7.7962 : L.r. 2.2500e-07
[task 2024-07-10T20:43:55.088Z] [2024-07-10 20:43:55] Ep. 1 : Up. 13 : Sen. 27,256 : Cost 7.26032591 : Time 0.25s : 218866.65 words/s : gNorm 7.8372 : L.r. 2.4375e-07
[task 2024-07-10T20:43:55.642Z] [2024-07-10 20:43:55] Ep. 1 : Up. 14 : Sen. 29,056 : Cost 7.24535465 : Time 0.55s : 185203.87 words/s : gNorm 7.7529 : L.r. 2.6250e-07
[task 2024-07-10T20:43:56.220Z] [2024-07-10 20:43:56] Ep. 1 : Up. 15 : Sen. 30,232 : Cost 7.22703648 : Time 0.58s : 181347.60 words/s : gNorm 7.6867 : L.r. 2.8125e-07
[task 2024-07-10T20:43:56.718Z] [2024-07-10 20:43:56] Ep. 1 : Up. 16 : Sen. 31,408 : Cost 7.23716879 : Time 0.50s : 170879.02 words/s : gNorm 7.6358 : L.r. 3.0000e-07
[task 2024-07-10T20:43:57.080Z] [2024-07-10 20:43:57] Ep. 1 : Up. 17 : Sen. 32,750 : Cost 7.29348230 : Time 0.36s : 200123.34 words/s : gNorm 7.5882 : L.r. 3.1875e-07
[task 2024-07-10T20:43:57.625Z] [2024-07-10 20:43:57] Ep. 1 : Up. 18 : Sen. 34,550 : Cost 7.22978544 : Time 0.54s : 185599.23 words/s : gNorm 7.5391 : L.r. 3.3750e-07
[task 2024-07-10T20:43:58.233Z] [2024-07-10 20:43:58] Ep. 1 : Up. 19 : Sen. 36,086 : Cost 7.29841566 : Time 0.61s : 144129.13 words/s : gNorm 7.5353 : L.r. 3.5625e-07
[task 2024-07-10T20:43:58.608Z] [2024-07-10 20:43:58] Ep. 1 : Up. 20 : Sen. 37,262 : Cost 7.22956038 : Time 0.38s : 110382.45 words/s : gNorm 7.4785 : L.r. 3.7500e-07
[task 2024-07-10T20:43:59.195Z] [2024-07-10 20:43:59] Ep. 1 : Up. 21 : Sen. 38,798 : Cost 7.22508717 : Time 0.59s : 162417.20 words/s : gNorm 7.4161 : L.r. 3.9375e-07
[task 2024-07-10T20:43:59.707Z] [2024-07-10 20:43:59] Ep. 1 : Up. 22 : Sen. 40,134 : Cost 7.22028923 : Time 0.51s : 159394.38 words/s : gNorm 7.4098 : L.r. 4.1250e-07
[task 2024-07-10T20:44:00.223Z] [2024-07-10 20:44:00] Ep. 1 : Up. 23 : Sen. 41,670 : Cost 7.23090315 : Time 0.52s : 154779.06 words/s : gNorm 7.4505 : L.r. 4.3125e-07
[task 2024-07-10T20:44:00.764Z] [2024-07-10 20:44:00] Ep. 1 : Up. 24 : Sen. 45,038 : Cost 7.18546581 : Time 0.54s : 180872.22 words/s : gNorm 7.4411 : L.r. 4.5000e-07
[task 2024-07-10T20:44:01.049Z] [2024-07-10 20:44:01] Ep. 1 : Up. 25 : Sen. 46,373 : Cost 7.23802185 : Time 0.28s : 230229.03 words/s : gNorm 7.4108 : L.r. 4.6875e-07
[task 2024-07-10T20:44:01.052Z] [2024-07-10 20:44:01] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz
[task 2024-07-10T20:44:02.619Z] [2024-07-10 20:44:02] Saving Adam parameters
[task 2024-07-10T20:44:05.588Z] [2024-07-10 20:44:05] [training] Saving training checkpoint to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz and /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.optimizer.npz
[task 2024-07-10T20:44:08.438Z] [2024-07-10 20:44:08] Ep. 1 : Up. 26 : Sen. 47,709 : Cost 7.22534895 : Time 7.39s : 9763.45 words/s : gNorm 7.3985 : L.r. 4.8750e-07
[task 2024-07-10T20:44:08.753Z] [2024-07-10 20:44:08] Ep. 1 : Up. 27 : Sen. 49,045 : Cost 7.24098825 : Time 0.31s : 130242.18 words/s : gNorm 7.4190 : L.r. 5.0625e-07
[task 2024-07-10T20:44:09.344Z] [2024-07-10 20:44:09] Ep. 1 : Up. 28 : Sen. 50,581 : Cost 7.19822454 : Time 0.59s : 184467.09 words/s : gNorm 7.3865 : L.r. 5.2500e-07
[task 2024-07-10T20:44:09.932Z] [2024-07-10 20:44:09] Ep. 1 : Up. 29 : Sen. 52,117 : Cost 7.15525723 : Time 0.59s : 175117.70 words/s : gNorm 7.3799 : L.r. 5.4375e-07
[task 2024-07-10T20:44:10.464Z] [2024-07-10 20:44:10] Ep. 1 : Up. 30 : Sen. 53,293 : Cost 7.15715647 : Time 0.53s : 170502.11 words/s : gNorm 7.3680 : L.r. 5.6250e-07
[task 2024-07-10T20:44:11.026Z] [2024-07-10 20:44:11] Ep. 1 : Up. 31 : Sen. 54,829 : Cost 7.14588070 : Time 0.56s : 183266.04 words/s : gNorm 7.3554 : L.r. 5.8125e-07
[task 2024-07-10T20:44:11.591Z] [2024-07-10 20:44:11] Ep. 1 : Up. 32 : Sen. 56,165 : Cost 7.15609503 : Time 0.57s : 179661.28 words/s : gNorm 7.3459 : L.r. 6.0000e-07
[task 2024-07-10T20:44:12.136Z] [2024-07-10 20:44:12] Ep. 1 : Up. 33 : Sen. 58,797 : Cost 7.14801073 : Time 0.55s : 197926.83 words/s : gNorm 7.3230 : L.r. 6.1875e-07
[task 2024-07-10T20:44:12.723Z] [2024-07-10 20:44:12] Ep. 1 : Up. 34 : Sen. 62,165 : Cost 7.11128283 : Time 0.59s : 183849.97 words/s : gNorm 7.4002 : L.r. 6.3750e-07
[task 2024-07-10T20:44:13.178Z] [2024-07-10 20:44:13] Ep. 1 : Up. 35 : Sen. 63,341 : Cost 7.11309004 : Time 0.46s : 172326.48 words/s : gNorm 7.3598 : L.r. 6.5625e-07
[task 2024-07-10T20:44:13.647Z] [2024-07-10 20:44:13] Ep. 1 : Up. 36 : Sen. 64,517 : Cost 7.06124067 : Time 0.47s : 148276.32 words/s : gNorm 7.3297 : L.r. 6.7500e-07
[task 2024-07-10T20:44:14.151Z] [2024-07-10 20:44:14] Ep. 1 : Up. 37 : Sen. 65,693 : Cost 7.10890532 : Time 0.50s : 144754.65 words/s : gNorm 7.2985 : L.r. 6.9375e-07
[task 2024-07-10T20:44:14.693Z] [2024-07-10 20:44:14] Ep. 1 : Up. 38 : Sen. 67,837 : Cost 7.10828733 : Time 0.54s : 177977.04 words/s : gNorm 7.2599 : L.r. 7.1250e-07
[task 2024-07-10T20:44:15.199Z] [2024-07-10 20:44:15] Ep. 1 : Up. 39 : Sen. 69,013 : Cost 7.12328482 : Time 0.51s : 132586.14 words/s : gNorm 7.2473 : L.r. 7.3125e-07
[task 2024-07-10T20:44:15.585Z] [2024-07-10 20:44:15] Ep. 1 : Up. 40 : Sen. 70,189 : Cost 7.00497437 : Time 0.39s : 127731.12 words/s : gNorm 7.2507 : L.r. 7.5000e-07
[task 2024-07-10T20:44:15.951Z] [2024-07-10 20:44:15] Ep. 1 : Up. 41 : Sen. 71,525 : Cost 7.04980755 : Time 0.37s : 135397.66 words/s : gNorm 7.2374 : L.r. 7.6875e-07
[task 2024-07-10T20:44:16.498Z] [2024-07-10 20:44:16] Ep. 1 : Up. 42 : Sen. 74,157 : Cost 7.03549337 : Time 0.55s : 144471.63 words/s : gNorm 7.2060 : L.r. 7.8750e-07
[task 2024-07-10T20:44:17.063Z] [2024-07-10 20:44:17] Ep. 1 : Up. 43 : Sen. 75,333 : Cost 7.01642370 : Time 0.57s : 170473.08 words/s : gNorm 7.1874 : L.r. 8.0625e-07
[task 2024-07-10T20:44:17.635Z] [2024-07-10 20:44:17] Ep. 1 : Up. 44 : Sen. 76,509 : Cost 7.03752041 : Time 0.57s : 152503.39 words/s : gNorm 7.1454 : L.r. 8.2500e-07
[task 2024-07-10T20:44:18.174Z] [2024-07-10 20:44:18] Ep. 1 : Up. 45 : Sen. 79,055 : Cost 7.02689838 : Time 0.54s : 179518.09 words/s : gNorm 7.1229 : L.r. 8.4375e-07
[task 2024-07-10T20:44:18.764Z] [2024-07-10 20:44:18] Ep. 1 : Up. 46 : Sen. 80,231 : Cost 7.02547598 : Time 0.59s : 179476.50 words/s : gNorm 7.0806 : L.r. 8.6250e-07
[task 2024-07-10T20:44:19.323Z] [2024-07-10 20:44:19] Ep. 1 : Up. 47 : Sen. 81,767 : Cost 7.01474953 : Time 0.56s : 164847.88 words/s : gNorm 7.0336 : L.r. 8.8125e-07
[task 2024-07-10T20:44:19.830Z] [2024-07-10 20:44:19] Ep. 1 : Up. 48 : Sen. 83,103 : Cost 6.95180702 : Time 0.51s : 152863.31 words/s : gNorm 7.0061 : L.r. 9.0000e-07
[task 2024-07-10T20:44:20.271Z] [2024-07-10 20:44:20] Ep. 1 : Up. 49 : Sen. 84,279 : Cost 6.96415138 : Time 0.44s : 139125.08 words/s : gNorm 7.0065 : L.r. 9.1875e-07
[task 2024-07-10T20:44:20.822Z] [2024-07-10 20:44:20] Ep. 1 : Up. 50 : Sen. 86,423 : Cost 6.90912437 : Time 0.55s : 155679.56 words/s : gNorm 6.9861 : L.r. 9.3750e-07
[task 2024-07-10T20:44:20.826Z] [2024-07-10 20:44:20] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz
[task 2024-07-10T20:44:23.619Z] [2024-07-10 20:44:23] Saving Adam parameters
[task 2024-07-10T20:44:26.638Z] [2024-07-10 20:44:26] [training] Saving training checkpoint to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz and /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.optimizer.npz
[task 2024-07-10T20:44:34.599Z] [2024-07-10 20:44:34] Training finished
[task 2024-07-10T20:44:40.717Z] [2024-07-10 20:44:40] [valid] First sentence's tokens as scored:
[task 2024-07-10T20:44:40.718Z] [2024-07-10 20:44:40] [valid] Decoding validation set with SentencePieceVocab for scoring
[task 2024-07-10T20:44:40.719Z] [2024-07-10 20:44:40] [valid]   Hyp: й й 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 й 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3 3
[task 2024-07-10T20:44:40.719Z] [2024-07-10 20:44:40] [valid]   Ref: Z E N I T : L U N E V , N E T O , S M O L N I K O V , I V A N O V I C H , N A B I U L L I N ( Z A B O L O T N Y , 8 3 ) , E R O K H I N , S H A T O V ( K U Z Y A E V , 4 6 ) , P A R E D E S , M A R C H I S I O ( K O K O R I N , 6 1 ) , D R I U S S I , D Z Y U B A .
[task 2024-07-10T20:48:34.845Z] [2024-07-10 20:48:34] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-chrf.npz
[task 2024-07-10T20:48:35.465Z] [2024-07-10 20:48:35] [valid] Ep. 1 : Up. 50 : chrf : 0.689894 : new best
[task 2024-07-10T20:48:36.316Z] [2024-07-10 20:48:36] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-ce-mean-words.npz
[task 2024-07-10T20:48:36.974Z] [2024-07-10 20:48:36] [valid] Ep. 1 : Up. 50 : ce-mean-words : 7.05813 : new best
[task 2024-07-10T20:52:12.155Z] [2024-07-10 20:52:12] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-bleu-detok.npz
[task 2024-07-10T20:52:12.795Z] [2024-07-10 20:52:12] [valid] Ep. 1 : Up. 50 : bleu-detok : 0 : new best
[task 2024-07-10T20:52:12.802Z] [2024-07-10 20:52:12] Saving model weights and runtime parameters to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz
[task 2024-07-10T20:52:15.380Z] [2024-07-10 20:52:15] Saving Adam parameters
[task 2024-07-10T20:52:18.040Z] [2024-07-10 20:52:18] [training] Saving training checkpoint to /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz and /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.optimizer.npz
[task 2024-07-10T20:52:28.738Z] [tracking INFO] Successfully parsed 305 lines
[task 2024-07-10T20:52:28.738Z] [tracking INFO] Found 50 training entries
[task 2024-07-10T20:52:28.738Z] [tracking INFO] Found 1 validation entries
[task 2024-07-10T20:52:28.841Z] + cp /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-chrf.npz /home/ubuntu/tasks/task_172064391152108/artifacts/final.model.npz.best-chrf.npz
[task 2024-07-10T20:52:29.005Z] + cp /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-chrf.npz.decoder.yml /home/ubuntu/tasks/task_172064391152108/artifacts/final.model.npz.best-chrf.npz.decoder.yml
[task 2024-07-10T20:52:29.007Z] + echo '### Model training is completed: /home/ubuntu/tasks/task_172064391152108/artifacts'
[task 2024-07-10T20:52:29.007Z] ### Model training is completed: /home/ubuntu/tasks/task_172064391152108/artifacts
[task 2024-07-10T20:52:29.007Z] + echo '###### Done: Training a model'
[task 2024-07-10T20:52:29.007Z] ###### Done: Training a model
[fetches 2024-07-10T20:52:29.025Z] removing /home/ubuntu/tasks/task_172064391152108/fetches
[fetches 2024-07-10T20:52:29.372Z] finished
[taskcluster 2024-07-10T20:52:29.384Z]    Exit Code: 0
[taskcluster 2024-07-10T20:52:29.384Z]    User Time: 1h3m21.054124s
[taskcluster 2024-07-10T20:52:29.384Z]  Kernel Time: 1m5.048255s
[taskcluster 2024-07-10T20:52:29.384Z]    Wall Time: 13m15.317774525s
[taskcluster 2024-07-10T20:52:29.384Z]       Result: SUCCEEDED
[taskcluster 2024-07-10T20:52:29.384Z] === Task Finished ===
[taskcluster 2024-07-10T20:52:29.384Z] Task Duration: 13m15.319930048s
[taskcluster 2024-07-10T20:52:29.417Z] Uploading artifact public/build/valid.log from file /home/ubuntu/tasks/task_172064391152108/artifacts/valid.log with content encoding "gzip", mime type "text/plain" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.419Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz.decoder.yml from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-bleu-detok.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.419Z] Uploading artifact public/build/model.npz.best-bleu-detok.npz from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-bleu-detok.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.420Z] Uploading artifact public/build/model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-chrf.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.421Z] Uploading artifact public/build/config.opustrainer.yml from file /home/ubuntu/tasks/task_172064391152108/artifacts/config.opustrainer.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.422Z] Uploading artifact public/build/final.model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_172064391152108/artifacts/final.model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.423Z] Uploading artifact public/build/vocab.spm from file /home/ubuntu/tasks/task_172064391152108/artifacts/vocab.spm with content encoding "gzip", mime type "application/x-source-rpm" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.424Z] Uploading artifact public/build/train.log from file /home/ubuntu/tasks/task_172064391152108/artifacts/train.log with content encoding "gzip", mime type "text/plain" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.425Z] Uploading artifact public/build/model.npz.decoder.yml from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.427Z] Uploading artifact public/build/model.npz.progress.yml from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.progress.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.431Z] Uploading artifact public/build/devset.out from file /home/ubuntu/tasks/task_172064391152108/artifacts/devset.out with content encoding "gzip", mime type "application/octet-stream" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.431Z] Uploading artifact public/build/model.npz.best-chrf.npz.decoder.yml from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-chrf.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.438Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-ce-mean-words.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.439Z] Uploading artifact public/build/opustrainer.log from file /home/ubuntu/tasks/task_172064391152108/artifacts/opustrainer.log with content encoding "gzip", mime type "text/plain" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.439Z] Uploading artifact public/build/model.npz.optimizer.npz from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.optimizer.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.439Z] Uploading artifact public/build/final.model.npz.best-chrf.npz from file /home/ubuntu/tasks/task_172064391152108/artifacts/final.model.npz.best-chrf.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.439Z] Uploading artifact public/build/model.npz from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz with content encoding "identity", mime type "application/octet-stream" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.452Z] Uploading artifact public/build/config.opustrainer.yml.state from file /home/ubuntu/tasks/task_172064391152108/artifacts/config.opustrainer.yml.state with content encoding "gzip", mime type "application/octet-stream" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.454Z] Uploading artifact public/build/model.npz.yml from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:29.469Z] Uploading artifact public/build/model.npz.best-ce-mean-words.npz.decoder.yml from file /home/ubuntu/tasks/task_172064391152108/artifacts/model.npz.best-ce-mean-words.npz.decoder.yml with content encoding "gzip", mime type "application/x-yaml" and expiry 2025-07-04T19:15:51.917Z
[taskcluster 2024-07-10T20:52:41.291Z] [mounts] Preserving cache: Moving "/home/ubuntu/tasks/task_172064391152108/checkouts" to "/home/ubuntu/caches/JGeQ3e-5TzmOm2wyxKUKUA"
[taskcluster 2024-07-10T20:52:41.346Z] Uploading link artifact public/logs/live.log to artifact public/logs/live_backing.log with expiry 2025-07-04T19:15:51.917Z