mf

ir-measures/Dockerfile

@@ -0,0 +1,15 @@
+FROM python:3.10
+
+RUN pip3 install ir-measures approvaltests pytest
+
+COPY ir_measures_evaluator.py /
+
+COPY tests /tmp/tests/
+
+RUN cd /tmp \
+        && find -iname __pycache__ -exec rm -Rf {} \; || echo "" \
+        && PYTHONPATH='../:.' pytest \
+        && cd / \
+        && rm -Rf /tmp/tests
+
+ENTRYPOINT [ "/ir_measures_evaluator.py" ]

ir-measures/Makefile

@@ -0,0 +1,15 @@
+IMAGE_VERSION=1.0.5
+
+build-docker-image:
+	docker build -t webis/ir_measures_evaluator:${IMAGE_VERSION} .
+
+.PHONY: tests
+tests:
+	pytest
+
+example-execution:
+	rm -Rf output
+	docker run --rm -it -v ${PWD}/input:/input -v ${PWD}/output:/output webis/ir_measures_evaluator:${IMAGE_VERSION} --run /input/run.txt --qrels /input/qrels.txt --measures "AP(rel=2)" "P(rel=2)@10" --output_path /output/eval.prototext
+
+publish-docker-image:
+	docker push webis/ir_measures_evaluator:${IMAGE_VERSION}

ir-measures/Pipfile

@@ -0,0 +1,14 @@
+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+ir-measures = "*"
+pytest = "*"
+approvaltests = "*"
+
+[dev-packages]
+
+[requires]
+python_version = "3.9"

ir-measures/README.md

@@ -0,0 +1,21 @@
+
+Add the evaluator to tira with:
+
+Image:
+```
+webis/ir_measures_evaluator:1.0
+```
+
+Command (if no qrels are available):
+
+```
+/ir_measures_evaluator.py --run ${inputRun}/run.txt --output_path ${outputDir}/evaluation.prototext
+```
+
+
+Command (if qrels are available):
+
+```
+/ir_measures_evaluator.py --run ${inputRun}/run.txt --topics ${inputDataset}/queries.jsonl --qrels ${inputDataset}/qrels.txt --output_path ${outputDir} --measures "P@10" "nDCG@10" "MRR"
+```
+

ir-measures/ir-measures.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="PYTHON_MODULE" version="4">
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="jdk" jdkName="Pipenv (ir-measures)" jdkType="Python SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>