Skip to content

Commit 11b0857

Browse files
ChristianDe-fabolousChristianDe-fabolous
committed
final submission
1 parent eede0d9 commit 11b0857

File tree

4 files changed

+64646
-50
lines changed

4 files changed

+64646
-50
lines changed

‎Dockerfile

-1
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ FROM webis/ir-lab-wise-2023:0.0.4
55
RUN pip3 uninstall -y tira \
66
&& pip3 install tira \
77
&& pip install openai \
8-
&& pip install nltk \
98
&& pip install pandas \
109
&& pip install nltk
1110

‎submission/initial-evaluation.ipynb

+58-49
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
"cells": [
33
{
44
"cell_type": "code",
5-
"execution_count": 37,
5+
"execution_count": 5,
66
"metadata": {},
77
"outputs": [],
88
"source": [
@@ -15,7 +15,7 @@
1515
},
1616
{
1717
"cell_type": "code",
18-
"execution_count": 38,
18+
"execution_count": 6,
1919
"metadata": {},
2020
"outputs": [],
2121
"source": [
@@ -26,7 +26,7 @@
2626
},
2727
{
2828
"cell_type": "code",
29-
"execution_count": 39,
29+
"execution_count": 7,
3030
"metadata": {},
3131
"outputs": [],
3232
"source": [
@@ -37,7 +37,7 @@
3737
},
3838
{
3939
"cell_type": "code",
40-
"execution_count": 41,
40+
"execution_count": 8,
4141
"metadata": {},
4242
"outputs": [
4343
{
@@ -94,6 +94,18 @@
9494
" </tr>\n",
9595
" <tr>\n",
9696
" <th>1</th>\n",
97+
" <td>bm25_auf_ngrams</td>\n",
98+
" <td>0.297068</td>\n",
99+
" <td>0.522970</td>\n",
100+
" <td>0.706496</td>\n",
101+
" <td>62893.0</td>\n",
102+
" <td>1354.0</td>\n",
103+
" <td>890.0</td>\n",
104+
" <td>68.0</td>\n",
105+
" <td>0.188954</td>\n",
106+
" </tr>\n",
107+
" <tr>\n",
108+
" <th>2</th>\n",
97109
" <td>Abbrevation</td>\n",
98110
" <td>0.367477</td>\n",
99111
" <td>0.568112</td>\n",
@@ -105,18 +117,6 @@
105117
" <td>0.254886</td>\n",
106118
" </tr>\n",
107119
" <tr>\n",
108-
" <th>2</th>\n",
109-
" <td>Ngrams</td>\n",
110-
" <td>0.352402</td>\n",
111-
" <td>0.567611</td>\n",
112-
" <td>0.717923</td>\n",
113-
" <td>62878.0</td>\n",
114-
" <td>1354.0</td>\n",
115-
" <td>908.0</td>\n",
116-
" <td>68.0</td>\n",
117-
" <td>0.226362</td>\n",
118-
" </tr>\n",
119-
" <tr>\n",
120120
" <th>3</th>\n",
121121
" <td>Only Ngrams</td>\n",
122122
" <td>0.294209</td>\n",
@@ -130,70 +130,79 @@
130130
" </tr>\n",
131131
" <tr>\n",
132132
" <th>4</th>\n",
133-
" <td>Old Ngram</td>\n",
134-
" <td>0.075044</td>\n",
135-
" <td>0.121938</td>\n",
136-
" <td>0.267352</td>\n",
137-
" <td>30915.0</td>\n",
138-
" <td>730.0</td>\n",
139-
" <td>287.0</td>\n",
140-
" <td>41.0</td>\n",
141-
" <td>0.058322</td>\n",
142-
" </tr>\n",
143-
" <tr>\n",
144-
" <th>5</th>\n",
145133
" <td>Full</td>\n",
146-
" <td>0.289385</td>\n",
147-
" <td>0.514831</td>\n",
134+
" <td>0.291334</td>\n",
135+
" <td>0.508704</td>\n",
148136
" <td>0.707547</td>\n",
149-
" <td>65118.0</td>\n",
137+
" <td>65120.0</td>\n",
150138
" <td>1354.0</td>\n",
151139
" <td>892.0</td>\n",
152140
" <td>68.0</td>\n",
153-
" <td>0.180295</td>\n",
141+
" <td>0.180148</td>\n",
142+
" </tr>\n",
143+
" <tr>\n",
144+
" <th>5</th>\n",
145+
" <td>NEU</td>\n",
146+
" <td>0.352402</td>\n",
147+
" <td>0.567611</td>\n",
148+
" <td>0.717923</td>\n",
149+
" <td>62878.0</td>\n",
150+
" <td>1354.0</td>\n",
151+
" <td>908.0</td>\n",
152+
" <td>68.0</td>\n",
153+
" <td>0.226362</td>\n",
154154
" </tr>\n",
155155
" </tbody>\n",
156156
"</table>\n",
157157
"</div>"
158158
],
159159
"text/plain": [
160-
" name ndcg_cut.10 recip_rank recall_1000 num_ret num_rel \\\n",
161-
"0 BM25 0.374041 0.579877 0.825376 66283.0 1354.0 \n",
162-
"1 Abbrevation 0.367477 0.568112 0.825376 68000.0 1354.0 \n",
163-
"2 Ngrams 0.352402 0.567611 0.717923 62878.0 1354.0 \n",
164-
"3 Only Ngrams 0.294209 0.512157 0.707547 62893.0 1354.0 \n",
165-
"4 Old Ngram 0.075044 0.121938 0.267352 30915.0 730.0 \n",
166-
"5 Full 0.289385 0.514831 0.707547 65118.0 1354.0 \n",
160+
" name ndcg_cut.10 recip_rank recall_1000 num_ret num_rel \\\n",
161+
"0 BM25 0.374041 0.579877 0.825376 66283.0 1354.0 \n",
162+
"1 bm25_auf_ngrams 0.297068 0.522970 0.706496 62893.0 1354.0 \n",
163+
"2 Abbrevation 0.367477 0.568112 0.825376 68000.0 1354.0 \n",
164+
"3 Only Ngrams 0.294209 0.512157 0.707547 62893.0 1354.0 \n",
165+
"4 Full 0.291334 0.508704 0.707547 65120.0 1354.0 \n",
166+
"5 NEU 0.352402 0.567611 0.717923 62878.0 1354.0 \n",
167167
"\n",
168168
" num_rel_ret num_q map \n",
169169
"0 1101.0 68.0 0.262311 \n",
170-
"1 1101.0 68.0 0.254886 \n",
171-
"2 908.0 68.0 0.226362 \n",
170+
"1 890.0 68.0 0.188954 \n",
171+
"2 1101.0 68.0 0.254886 \n",
172172
"3 892.0 68.0 0.186284 \n",
173-
"4 287.0 41.0 0.058322 \n",
174-
"5 892.0 68.0 0.180295 "
173+
"4 892.0 68.0 0.180148 \n",
174+
"5 908.0 68.0 0.226362 "
175175
]
176176
},
177-
"execution_count": 41,
177+
"execution_count": 8,
178178
"metadata": {},
179179
"output_type": "execute_result"
180180
}
181181
],
182182
"source": [
183183
"# This assumes we have execited the ../baseline-retrieval-system/baseline-retrieval-system.ipynb notebook before.\n",
184+
"bm25_auf_ngrams = bm25 = pt.io.read_results('./runs/run.txt')\n",
184185
"bm25 = pt.io.read_results('./runs/bm25.txt')\n",
185186
"abbrevations = pt.io.read_results('./runs/abbrevations.txt')\n",
186-
"ngrams = pt.io.read_results('./runs/run_querys.txt')\n",
187-
"ngrams2 = pt.io.read_results('./runs/run.txt')\n",
187+
"#ngrams = pt.io.read_results('./runs/run_querys.txt')\n",
188+
"#ngrams2 = pt.io.read_results('./runs/run.txt')\n",
188189
"only_ngrams = pt.io.read_results('./runs/onlyngram/run.txt')\n",
189-
"old_ngram = pt.io.read_results('./runs/oldngram/run.txt')\n",
190+
"#old_ngram = pt.io.read_results('./runs/oldngram/run.txt')\n",
190191
"full = pt.io.read_results('./runs/fullrun/run.txt')\n",
192+
"#b02 = pt.io.read_results('./runs/fullrun/0.2/run.txt')\n",
193+
"#b05 = pt.io.read_results('./runs/fullrun/0.5/run.txt')\n",
194+
"#b75 = pt.io.read_results('./runs/fullrun/run.txt')\n",
195+
"\n",
196+
"#ngram_b02 = pt.io.read_results('./runs/fullrun/0.2/run.txt')\n",
197+
"#ngram_b05 = pt.io.read_results('./runs/fullrun/0.5/run.txt')\n",
198+
"#ngram_b75 = pt.io.read_results('./runs/fullrun/run.txt')\n",
199+
"neues_notebook = pt.io.read_results('./runs/fullrun/submission/run.txt')\n",
191200
"pt.Experiment(\n",
192-
" [bm25, abbrevations, ngrams, only_ngrams, old_ngram, full],\n",
201+
" [bm25,bm25_auf_ngrams, abbrevations, only_ngrams,full, neues_notebook],\n",
193202
" pt_dataset.get_topics(),\n",
194203
" pt_dataset.get_qrels(),\n",
195204
" [\"ndcg_cut.10\", \"recip_rank\", \"recall_1000\", \"num_ret\", \"num_rel\",\"num_rel_ret\", \"num_q\", \"map\"],\n",
196-
" names=[\"BM25\", \"Abbrevation\", \"Ngrams\", \"Only Ngrams\", \"Old Ngram\", \"Full\"]\n",
205+
" names=[\"BM25\",\"bm25_auf_ngrams\", \"Abbrevation\", \"Only Ngrams\", \"Full\", \"NEU\" ]\n",
197206
")"
198207
]
199208
}

0 commit comments

Comments
 (0)