-
Notifications
You must be signed in to change notification settings - Fork 11
/
03_run_eval.sh
executable file
·185 lines (141 loc) · 9.8 KB
/
03_run_eval.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
#!/bin/bash
# FILES=/Users/rebeiro/github/yoruba-adr/models/*.pt
# ========================================================================================
# Accuracy (%): 40.30612244897959
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_256_1_1_step_100000.pt
# (September 2018/March 2019) original 3 corpus Biblica, Blog, LagosNWU
# CORPUS: {LagosNWU, Bíbélì & Yorùbá blog}
# MODEL: yo_adr_bahdanau_lstm_256_1_1_step_100000.pt
# GITHUB:
# NFC OK!!
# ========================================================================================
# Accuracy (%): 57.29068673565381
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_iroyin_step_21000_no_kola_mistake.pt
# (Oct 16, 2019) No Kola mistake model -- NO IROYIN
# CORPUS: {LagosNWUspeech_corpus, TheYorubaBlog_corpus, BibeliYoruba_corpus, Toluwase/Word-Level-Language-Identification-for-Resource-Scarce-, [MISSING] Kọ́lá Túbọ̀sún interiews}
# MODEL: yo_adr_bahdanau_lstm_128_2_2_sans_iroyin_step_21000_no_kola_mistake.pt
# GITHUB: https://github.com/Niger-Volta-LTI/yoruba-adr/blob/19cab1e45683e74cfe67e58b118d8d8e72368f6e/runs/onmt/run_results_with_source_corpora.txt
# NFC OK!!
# ========================================================================================
# Accuracy (%): 58.203696316979126
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_iroyin_step_22500_keeper.pt
# (Nov 6th, 2019) Good model what we originally intended 22500 -- NO IROYIN
# CORPUS: {LagosNWUspeech_corpus, TheYorubaBlog_corpus, BibeliYoruba_corpus, Toluwase/Word-Level-Language-Identification-for-Resource-Scarce-, Kọ́lá Túbọ̀sún interiews}
# MODEL: yo_adr_bahdanau_lstm_128_2_2_sans_iroyin_step_22500_keeper.pt (eval'd to give the best results)
# GITHUB: https://github.com/Niger-Volta-LTI/yoruba-adr/blob/3fd97403aeb9da70e5c218feafc7d1c4edd81fed/runs/onmt/run_results_with_source_corpora.txt
# NFC OK!! because Kola's second interview isn't in here yet
# ========================================================================================
# Accuracy (%): 72.43340918217106
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_100000.pt
# Accuracy (%): 72.97166216309844
#FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_79500.pt
# Accuracy (%): 73.04896447675216
#FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_80500.pt
# Accuracy (%): 72.37981881901739
#FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_95000.pt
#Accuracy (%): 72.48129342597541
#FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_92500.pt
# Accuracy (%): 72.51117431938236
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_91500.pt
# Accuracy (%): 72.69489247311827
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_91000.pt
#Accuracy (%): 72.30301427815971
#FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_85000.pt
# Accuracy (%): 72.47736954206603
#FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_87500.pt
#Accuracy (%): 72.134166392634
#FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_88500.pt
# Accuracy (%): 72.87628375161532
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_89500.pt
# TOPBOY ############
# Accuracy (%): 73.34699453551913
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_90000.pt
# Accuracy (%): 73.0543470754782
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan30_EC2_step_90500.pt
# ========================================================================================
############
# bahdanau SANS JW300
# Accuracy (%): 62.872217403873954
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_40000.pt
# Accuracy (%): 62.28550018733608
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_10000.pt
# Accuracy (%): 63.63151884278645
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_20000.pt
# Accuracy (%): 62.519213034122345
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_22500.pt
# Accuracy (%): 63.86027418480366
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_23000.pt
# TOPBOY ############
# Accuracy (%): 64.88090589613432
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_23500.pt
# Accuracy (%): 63.52005782435851
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_24000.pt
# Accuracy (%): 63.985906862745104
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_25000.pt
# Accuracy (%): 62.33470467234793
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_27500.pt
# Accuracy (%): 62.20488747756454
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_30000.pt
# Accuracy (%): 62.49199459190209
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_jan31_no_JW300_EC2_step_26000.pt
# ========================================================================================
############
# TRANSFORMER SANS JW300
# Accuracy (%): 72.05630587787711
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb1_no_JW300_EC2_step_10000.pt
# Accuracy (%): 73.11475409836066
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb1_no_JW300_EC2_step_12000.pt
############ TOPBOY 1
# Accuracy (%): 73.99654311503745
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb1_no_JW300_EC2_step_12500.pt
#Accuracy (%): 71.15181401902079
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb1_no_JW300_EC2_step_13000.pt
# Accuracy (%): 73.87867647058823
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb1_no_JW300_EC2_step_13500.pt
# Accuracy (%): 72.00752393980848
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb1_no_JW300_EC2_step_14500.pt
############ TOPBOY
# Accuracy (%): 73.99738854691289
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb1_no_JW300_EC2_step_15500.pt
# Accuracy (%): 72.50434782608696
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb1_no_JW300_EC2_step_16000.pt
# ========================================================================================
# ========================================================================================
############
# TRANSFORMER ALL IN
# Accuracy (%): 77.8399552322328
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb2_EC2_step_13500.pt
############ TOPBOY
# Accuracy (%): 79.36962750716332
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb2_EC2_step_21500.pt
# Accuracy (%): 79.28860873480414
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb2_EC2_step_23500.pt
# Accuracy (%): 79.1218377435301
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb2_EC2_step_16500.pt
# Accuracy (%): 78.79742304939155
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb2_EC2_step_17500.pt
# ========================================================================================
############
# TRANSFORMER + FASTTEXT EMBEDDING
# FILES=/Users/iroro/github/yoruba-adr/models/embeddings_results_transformer_feb8/*.pt
############ TOPBOY
# Accuracy (%): 81.32315521628499
# /Users/iroro/github/yoruba-adr/models/embeddings_results_transformer_feb8/yo_adr_transformer_sans_yoglobalvoices_all_in_take3_feb8_EC2_step_18500.pt file...
# ========================================================================================
############
# bahdanau + FASTTEXT EMBEDDING
# FILES=/Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_feb9_fasttext_EC2_step_*.pt
# Processing /Users/iroro/github/yoruba-adr/models/yo_adr_bahdanau_lstm_128_2_2_sans_yoglobalvoices_all_in_take3_feb9_fasttext_EC2_step_93500.pt file...
# Accuracy (%): 71.10906645905585
for f in $FILES
do
echo "Processing $f file..."
# take action on each file. $f store current file name
python3 ./src/translate.py \
-model $f \
-src data/test/sources.txt \
-tgt data/test/targets.txt \
-output data/test/pred.txt \
-replace_unk \
-verbose
done