full_pipeline_evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from translation import BasicGPT4Translator, CustomQueryGPT4Translator, PROMPT_LIBRARY, PROMPT_LIBRARY_PL, PROMPT_LIBRARY_5_SHOT, CustomQueryLineFormatGPT4Translator, StoryTellingTranslator, MantraResultsLoadingTranslator, SceneAwareNMTResultsLoadingTranslator, ThreePageContextTranslator, CustomQueryLineFormatGPT4TranslatorJSON, CustomQueryLineFormatGPT4TranslatorJSONExampleImage, NoImageLineByLineGPT4Translator, LongFormContextTranslatorJSON, ThreePageContextTranslatorJSON, NoImageLineByLineGPT4TranslatorPolish, SelfRankingGPT4TranslatorJSON, ImageLineByLineGPT4Translator, WholeVolumeTranslatorJSON, VolumeByPageAllContextTranslator\n",
    "from text_extraction import TextExtractorPixelwise, OpenMantraAnnotationMatcher, LoveHinaAnnotationMatcher\n",
    "from test_running import OpenMantraLoader, TestRunner, OpenMantraLoaderModifiedImage, LoveHinaLoader\n",
    "from translation_evaluation import BleuEvaluator, ChrfEvaluator, TerEvaluator, MeteorEvaluator, CometEvaluator, BertScoreEvaluator, XCometEvaluator, XXLCometEvaluator\n",
    "from compression import LongFormCompression\n",
    "\n",
    "import os\n",
    "import json\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "import logging\n",
    "load_dotenv(find_dotenv())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S')\n",
    "\n",
    "logging.basicConfig(filename=f'logs/bin.log', filemode='a', format='%(name)s - %(levelname)s - %(message)s',datefmt='%H:%M:%S', level=logging.INFO, force=True)\n",
    "\n",
    "if os.environ.get(\"PRINT_LOGS\", 'False').lower() in ('true', '1', 't'):\n",
    "    logging.getLogger(\"Experiment\").setLevel(logging.DEBUG)\n",
    "else:\n",
    "    logging.getLogger(\"Experiment\").setLevel(logging.INFO)\n",
    "\n",
    "fh = logging.FileHandler(f'logs/experiment_{time_str}.log')\n",
    "\n",
    "logging.getLogger(\"Experiment\").addHandler(fh)\n",
    "\n",
    "logging.getLogger(\"Experiment\").info('Logger initiated')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "API_KEY = os.environ.get(\"GPT_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# text_extractor = TextExtractorPixelwise(path_to_model='../Manga-Text-Segmentation/model.pkl')\n",
    "# text_extractor = OpenMantraAnnotationMatcher(annotation_file_path='../open-mantra-dataset/annotation.json')\n",
    "text_extractor = LoveHinaAnnotationMatcher(annotation_file_path='../LoveHina_vol01_fixed_order_page_split.json')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "validation_set = [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), (0, 7), (0, 8), (0, 9), (0, 11), (0, 12), (0, 13), (0, 14), (0, 15), (0, 16), (0, 17), (0, 18), (0, 19), (0, 20), (0, 21), (0, 22), (0, 23), (0, 24), (0, 25), (0, 26), (0, 27), (0, 28), (0, 29), (0, 30), (0, 31), (0, 32), (0, 33), (0, 34), (0, 35), (0, 36), (0, 37), (0, 38), (0, 39), (0, 40), (0, 41), (0, 42), (0, 43), (0, 44), (0, 45), (1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), (1, 6), (1, 7), (1, 8), (1, 9), (1, 10), (1, 11), (1, 12), (1, 13), (1, 14), (1, 15), (1, 16), (1, 17), (1, 18), (1, 19), (1, 20), (1, 21), (1, 22), (1, 23), (1, 24), (1, 25), (1, 26), (1, 27), (1, 28), (1, 29), (1, 30), (1, 31), (1, 32), (1, 33), (1, 34), (1, 35), (1, 36), (1, 37)]\n",
    "\n",
    "test_set = [(2, 0), (2, 1), (2, 2), (2, 3), (2, 4), (2, 5), (2, 6), (2, 7), (2, 8), (2, 9), (2, 10), (2, 11), (2, 12), (2, 13), (2, 14), (2, 15), (2, 16), (2, 17), (2, 18), (2, 19), (2, 20), (2, 21), (2, 22), (2, 23), (2, 24), (2, 25), (2, 26), (2, 27), (2, 28), (2, 29), (2, 30), (2, 31), (2, 32), (2, 33), (2, 34), (2, 35), (2, 36), (2, 37), (2, 38), (2, 39), (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10), (3, 11), (3, 12), (3, 13), (3, 14), (3, 15), (3, 16), (3, 17), (3, 18), (3, 19), (3, 20), (3, 21), (3, 22), (3, 23), (3, 24), (3, 25), (3, 26), (3, 27), (3, 28), (3, 29), (3, 30), (3, 31), (3, 32), (3, 33), (3, 34), (3, 35), (4, 0), (4, 1), (4, 2), (4, 3), (4, 4), (4, 5), (4, 6), (4, 7), (4, 8), (4, 9), (4, 10), (4, 11), (4, 12), (4, 13), (4, 14), (4, 15), (4, 16), (4, 17), (4, 18), (4, 19), (4, 20), (4, 21), (4, 22), (4, 23), (4, 24), (4, 25), (4, 26), (4, 27), (4, 28), (4, 29), (4, 30), (4, 31), (4, 32), (4, 33), (4, 34), (4, 35), (4, 36), (4, 37), (4, 38), (4, 39), (4, 40), (4, 41), (4, 42), (4, 43), (4, 44), (4, 45), (4, 46), (4, 47), (4, 48), (4, 49), (4, 50), (4, 51), (4, 52), (4, 53)]\n",
    "\n",
    "all_set = [(0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), (0, 7), (0, 8), (0, 9), (0, 10), (0, 11), (0, 12), (0, 13), (0, 14), (0, 15), (0, 16), (0, 17), (0, 18), (0, 19), (0, 20), (0, 21), (0, 22), (0, 23), (0, 24), (0, 25), (0, 26), (0, 27), (0, 28), (0, 29), (0, 30), (0, 31), (0, 32), (0, 33), (0, 34), (0, 35), (0, 36), (0, 37), (0, 38), (0, 39), (0, 40), (0, 41), (0, 42), (0, 43), (0, 44), (0, 45), (1, 0), (1, 1), (1, 2), (1, 3), (1, 4), (1, 5), (1, 6), (1, 7), (1, 8), (1, 9), (1, 10), (1, 11), (1, 12), (1, 13), (1, 14), (1, 15), (1, 16), (1, 17), (1, 18), (1, 19), (1, 20), (1, 21), (1, 22), (1, 23), (1, 24), (1, 25), (1, 26), (1, 27), (1, 28), (1, 29), (1, 30), (1, 31), (1, 32), (1, 33), (1, 34), (1, 35), (1, 36), (1, 37), (2, 0), (2, 1), (2, 2), (2, 3), (2, 4), (2, 5), (2, 6), (2, 7), (2, 8), (2, 9), (2, 10), (2, 11), (2, 12), (2, 13), (2, 14), (2, 15), (2, 16), (2, 17), (2, 18), (2, 19), (2, 20), (2, 21), (2, 22), (2, 23), (2, 24), (2, 25), (2, 26), (2, 27), (2, 28), (2, 29), (2, 30), (2, 31), (2, 32), (2, 33), (2, 34), (2, 35), (2, 36), (2, 37), (2, 38), (2, 39), (3, 0), (3, 1), (3, 2), (3, 3), (3, 4), (3, 5), (3, 6), (3, 7), (3, 8), (3, 9), (3, 10), (3, 11), (3, 12), (3, 13), (3, 14), (3, 15), (3, 16), (3, 17), (3, 18), (3, 19), (3, 20), (3, 21), (3, 22), (3, 23), (3, 24), (3, 25), (3, 26), (3, 27), (3, 28), (3, 29), (3, 30), (3, 31), (3, 32), (3, 33), (3, 34), (3, 35), (4, 0), (4, 1), (4, 2), (4, 3), (4, 4), (4, 5), (4, 6), (4, 7), (4, 8), (4, 9), (4, 10), (4, 11), (4, 12), (4, 13), (4, 14), (4, 15), (4, 16), (4, 17), (4, 18), (4, 19), (4, 20), (4, 21), (4, 22), (4, 23), (4, 24), (4, 25), (4, 26), (4, 27), (4, 28), (4, 29), (4, 30), (4, 31), (4, 32), (4, 33), (4, 34), (4, 35), (4, 36), (4, 37), (4, 38), (4, 39), (4, 40), (4, 41), (4, 42), (4, 43), (4, 44), (4, 45), (4, 46), (4, 47), (4, 48), (4, 49), (4, 50), (4, 51), (4, 52), (4, 53)]\n",
    "\n",
    "love_hina_all_set = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 183, 184, 185, 186, 187, 188, 189, 190, 191, 192, 193]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "evaluators = [BleuEvaluator(), ChrfEvaluator(), MeteorEvaluator(), BertScoreEvaluator(), XXLCometEvaluator()]\n",
    "\n",
    "# evaluators = [BleuEvaluator()]\n",
    "\n",
    "# data_loader_modified = OpenMantraLoaderModifiedImage('../open-mantra-dataset', validation_set[0:3], test_set)\n",
    "\n",
    "# data_loader_unmodified = OpenMantraLoader('../open-mantra-dataset', validation_set[0:3], test_set)\n",
    "\n",
    "data_loader_unmodified = LoveHinaLoader('./LoveHina/LoveHina_vol01_page_split', '../LoveHina_vol01_fixed_order_page_split.json', love_hina_all_set[3:5], love_hina_all_set)\n",
    "\n",
    "data_loader_modified = LoveHinaLoader('./LoveHina/LoveHina_vol01_page_split_numbered', '../LoveHina_vol01_fixed_order_page_split.json', love_hina_all_set[3:5], love_hina_all_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#translator = BasicGPT4Translator(text_extractor, API_KEY)\n",
    "\n",
    "RETRY_LIMIT = 10\n",
    "\n",
    "TEMP = 0.5\n",
    "\n",
    "sys_msg = \"\"\"You are a manga translator.\"\"\"\n",
    "\n",
    "sys_msg_pl = \"\"\"Jesteś tłumaczem mang\"\"\"\n",
    "\n",
    "\n",
    "query_template_indexes_1 = ['line by line english image empowered translator']\n",
    "\n",
    "\n",
    "logging.getLogger(\"Experiment\").info(f'Testing prompts {query_template_indexes_1}')\n",
    "\n",
    "\n",
    "line_format = \"\"\"Line {}: {};\"\"\"\n",
    "\n",
    "line_format_pl = \"\"\"Zdanie {}: {};\"\"\"\n",
    "\n",
    "\n",
    "translators = []\n",
    "\n",
    "translators_img_unmodif = []\n",
    "\n",
    "# translators.append(\n",
    "#               SelfRankingGPT4TranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY[51], reranking_query=PROMPT_LIBRARY[52], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=0.5, skip_image=False)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             NoImageLineByLineGPT4Translator(text_extractor=text_extractor, api_key=API_KEY, lang = \"Polish\", lang_example = \"Dziękuję\")\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             NoImageLineByLineGPT4TranslatorPolish(text_extractor=text_extractor, api_key=API_KEY)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#         CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_5_SHOT[11], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=True)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#         CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[1102], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=True)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_5_SHOT[15], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[1502], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "# translators.append(\n",
    "#             CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_5_SHOT[22], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "# translators.append(\n",
    "#             CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[2202], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#         CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[1105], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=True)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[1505], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "\n",
    "# translators.append(\n",
    "#             CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[2205], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "translators.append(\n",
    "        CustomQueryLineFormatGPT4TranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[44], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "        )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             NoImageLineByLineGPT4Translator(text_extractor=text_extractor, api_key=API_KEY, lang=\"Polish\", lang_example=PROMPT_LIBRARY_PL['examples'])\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             ImageLineByLineGPT4Translator(text_extractor=text_extractor, api_key=API_KEY, lang=\"Polish\", lang_example=\"Example: Line 1: お腹すいたー Return: Translation 1: [Ale jestem głodnaaa](Osoba, która mówi to dziewczynka trzymająca się za brzuch. Dlatego stosujemy rodzaj żeński w tłumaczeniu.).\")\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             ImageLineByLineGPT4Translator(text_extractor=text_extractor, api_key=API_KEY, lang=\"Polish\", lang_example=PROMPT_LIBRARY_PL['examples_explained'])\n",
    "# )\n",
    "\n",
    "compresor = LongFormCompression(API_KEY, lang=\"Polish\")\n",
    "\n",
    "translators_img_unmodif.append(\n",
    "            LongFormContextTranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[55], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, context_handler=compresor, api_key=API_KEY, temperature=TEMP, skip_image=False, lang='pl')\n",
    "        )\n",
    "\n",
    "translators_img_unmodif.append(\n",
    "            LongFormContextTranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[5505], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, context_handler=compresor, api_key=API_KEY, temperature=TEMP, skip_image=False, lang='pl')\n",
    "        )\n",
    "\n",
    "translators.append(\n",
    "            ThreePageContextTranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[50], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    ")\n",
    "\n",
    "translators.append(\n",
    "            ThreePageContextTranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_PL[5005], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    ")\n",
    "\n",
    "\n",
    "#################################################################################\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             MantraResultsLoadingTranslator(text_extractor=text_extractor, api_key=API_KEY, data_path='../image_tag_vis.json')\n",
    "#         )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             SceneAwareNMTResultsLoadingTranslator(text_extractor=text_extractor, api_key=API_KEY, dataset_anotation_json_path='../open-mantra-dataset/annotation.json', data_path='../translation/Scene-aware_NMT/+genre', predict_to_use=1)\n",
    "# )\n",
    "# translators_img_unmodif.append(\n",
    "#             SceneAwareNMTResultsLoadingTranslator(text_extractor=text_extractor, api_key=API_KEY, dataset_anotation_json_path='../open-mantra-dataset/annotation.json', data_path='../translation/Scene-aware_NMT/+genre', predict_to_use=2)\n",
    "# )\n",
    "# translators_img_unmodif.append(\n",
    "#             SceneAwareNMTResultsLoadingTranslator(text_extractor=text_extractor, api_key=API_KEY, dataset_anotation_json_path='../open-mantra-dataset/annotation.json', data_path='../translation/Scene-aware_NMT/+genre', predict_to_use=3)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             NoImageLineByLineGPT4Translator(text_extractor=text_extractor, api_key=API_KEY, lang_example=PROMPT_LIBRARY_5_SHOT['examples'])\n",
    "# )\n",
    "\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             ImageLineByLineGPT4Translator(text_extractor=text_extractor, api_key=API_KEY, lang_example=PROMPT_LIBRARY_5_SHOT['examples_explained'])\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#         CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY[11], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=True)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY[15], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "#         )\n",
    "\n",
    "# translators.append(\n",
    "#             CustomQueryLineFormatGPT4Translator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY[22], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "#         )\n",
    "\n",
    "# translators.append(\n",
    "#         CustomQueryLineFormatGPT4TranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY[44], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "#         )\n",
    "\n",
    "# translators.append(\n",
    "#         CustomQueryLineFormatGPT4TranslatorJSONExampleImage(system_msg=sys_msg, query_txt=PROMPT_LIBRARY[46], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, example_imgs_paths =['./data/cashier_example_numbered_panels.jpg'] ,temperature=TEMP)\n",
    "# )\n",
    "\n",
    "# compresor = LongFormCompression(API_KEY)\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             LongFormContextTranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY[55], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, context_handler=compresor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "#         )\n",
    "\n",
    "# translators.append(\n",
    "#             ThreePageContextTranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_5_SHOT[50], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "# translators.append(\n",
    "#             WholeVolumeTranslatorJSON(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_5_SHOT[53], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             VolumeByPageAllContextTranslator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY[56], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "# translators_img_unmodif.append(\n",
    "#             VolumeByPageAllContextTranslator(system_msg=sys_msg, query_txt=PROMPT_LIBRARY_5_SHOT[56], line_passing_format=line_format, retry_limit=RETRY_LIMIT, text_extractor=text_extractor, api_key=API_KEY, temperature=TEMP, skip_image=False)\n",
    "# )\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "RUNS_TO_AVG_OVER = 1\n",
    "CONCAT_LINES = False\n",
    "RUN_VALIDATION = False\n",
    "DEST_LANG = 'pl' # 'en' #\n",
    "\n",
    "VOLUME_TRANSLATION = True\n",
    "\n",
    "total_scores = []\n",
    "\n",
    "for i, translator in enumerate(translators_img_unmodif):\n",
    "    print(\"*********************************************************\")\n",
    "    print(f\"                 TRANSLATOR {i+1}/{len(translators_img_unmodif)}\")\n",
    "    print(\"*********************************************************\")\n",
    "\n",
    "    logging.getLogger(\"Experiment\").info(f\"-------------------TRANSLATOR {i+1}/{len(translators_img_unmodif)}-------------------\")\n",
    "    if not VOLUME_TRANSLATION:\n",
    "        scores = TestRunner.test_page_translation(translator, data_loader_unmodified, evaluators, runs = RUNS_TO_AVG_OVER, concat_lines = CONCAT_LINES, validation_set=RUN_VALIDATION, dest_lang=DEST_LANG)\n",
    "    else:\n",
    "        scores = TestRunner.test_volume_translation(translator, data_loader_unmodified, evaluators, runs = RUNS_TO_AVG_OVER, concat_lines = CONCAT_LINES, validation_set=RUN_VALIDATION, dest_lang=DEST_LANG)\n",
    "    logging.getLogger(\"Experiment\").info(f\"-------------------TRANSLATOR {i+1}/{len(translators_img_unmodif)} FINAL SCORES -------------------\")\n",
    "    logging.getLogger(\"Experiment\").info(scores)\n",
    "    total_scores.append(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, translator in enumerate(translators):\n",
    "    print(\"*********************************************************\")\n",
    "    print(f\"                 TRANSLATOR {i+1}/{len(translators)}\")\n",
    "    print(\"*********************************************************\")\n",
    "\n",
    "    logging.getLogger(\"Experiment\").info(f\"-------------------TRANSLATOR {i+1}/{len(translators)}-------------------\")\n",
    "    if not VOLUME_TRANSLATION:\n",
    "        scores = TestRunner.test_page_translation(translator, data_loader_modified, evaluators, runs = RUNS_TO_AVG_OVER, concat_lines = CONCAT_LINES, validation_set=RUN_VALIDATION, dest_lang=DEST_LANG)\n",
    "    else:\n",
    "        scores = TestRunner.test_volume_translation(translator, data_loader_modified, evaluators, runs = RUNS_TO_AVG_OVER, concat_lines = CONCAT_LINES, validation_set=RUN_VALIDATION, dest_lang=DEST_LANG)\n",
    "    logging.getLogger(\"Experiment\").info(f\"-------------------TRANSLATOR {i+1}/{len(translators)} FINAL SCORES -------------------\")\n",
    "    logging.getLogger(\"Experiment\").info(scores)\n",
    "    total_scores.append(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.getLogger(\"Experiment\").info(f\"-------------------SCORES-------------------\")\n",
    "for scores in total_scores:\n",
    "    logging.getLogger(\"Experiment\").info(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(total_scores[0])):\n",
    "    for scores in total_scores:\n",
    "        print(scores[i])\n",
    "\n",
    "    print('----------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "logging.getLogger(\"Experiment\").info(f'The experiment ended at {datetime.now().strftime(\"%Y-%m-%d %H:%M:%S\")}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scoreses = []\n",
    "\n",
    "for i in range(len(total_scores[0])):\n",
    "    sum = 0\n",
    "\n",
    "    for scores in total_scores:\n",
    "        sum += scores[i][1]\n",
    "\n",
    "    tmp = (total_scores[0][i][0], round(sum/len(total_scores),3))\n",
    "    scoreses.append(tmp)\n",
    "\n",
    "print(scoreses)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}