notebooks/Demo-text-search-simple.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "6975e34a",
   "metadata": {},
   "source": [
    "# Text search in PageXML files\n",
    "\n",
    "There are built-in functions for:\n",
    "\n",
    "- keyword/keyphrase in context search\n",
    "- term similarity search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1d77f65f",
   "metadata": {},
   "outputs": [],
   "source": [
    "%reload_ext autoreload\n",
    "%autoreload 2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5649b53a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r1', 'line_id': 'r1l1', 'text': None}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l1', 'text': 'f. 1.'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l2', 'text': 'Inventaris ende beschrijvinge van'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l3', 'text': 'naerbeschreven Cleedinge van'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l4', 'text': 'Linne ende Wollen, Item huijsraet'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l5', 'text': 'naergelaten by Janneken de Veel'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l6', 'text': 'oude Vrijster alhier overleden;'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l7', 'text': 'die volgens hare dispositie van'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l8', 'text': \"uterste wille, door d' eersame\"}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l9', 'text': None}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l10', 'text': 'Sr. hans van Essen Bouckhouder'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l11', 'text': 'vande Westinde. Compe. alhier In'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l12', 'text': 'Amstm. versonden sullen werden'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l13', 'text': 'naer Ryssel aen Beatricx de'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l14', 'text': 'Veel suster vande voorn. Janneken'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l15', 'text': 'de Veel sa: Beschreven door'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l16', 'text': 'mij Henrick Schaeff Nots etc.'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l17', 'text': 'in presentie vande ondergess.'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l18', 'text': 'getuijgen. desen 1e. Januarij'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l19', 'text': '@ 1639.'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l20', 'text': None}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l21', 'text': 'vijff floppen.'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l22', 'text': 'Twaelf kragen soo goede als quade.'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l23', 'text': 'Twaelf slaeplakens. soo goede als quade'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l24', 'text': 'Twee oude brabantsche Vrouwen houven met hare'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l25', 'text': 'Custodien.'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l26', 'text': 'Ses Servetten met een ammelarij van dast damast werc'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l27', 'text': 'Sesthien Servetten van divertse bloem, soo goede'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l28', 'text': 'als quade'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l29', 'text': 'Vijff ommeslaeckens'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l30', 'text': 'Twee grote handtdoecken'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l31', 'text': 'een gestreepte borstrock'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l32', 'text': None}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l33', 'text': 'noch een Slaeplaken'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l34', 'text': 'Ses nacht halsdoeken soo quade als goede'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l35', 'text': 'darthien Fluwijnen, soo quade als goede'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l36', 'text': 'Elff hembden soo quade als goede'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l37', 'text': 'vyff witte voorschoden, als boven'}\n",
      "{'doc_id': 'a30768000008.jpg', 'textregion_id': 'r2', 'line_id': 'r2l38', 'text': 'een glase Lecksteen'}\n"
     ]
    }
   ],
   "source": [
    "from pagexml.parser import parse_pagexml_file\n",
    "import pagexml.helper.text_helper as text_helper\n",
    "\n",
    "data_dir = '../data'\n",
    "filepath = f'{data_dir}/example.xml'\n",
    "\n",
    "lines = text_helper.LineReader(pagexml_files=[filepath])\n",
    "\n",
    "for line in lines:\n",
    "    print(line)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d5837891",
   "metadata": {},
   "source": [
    "## Finding a term in the text of scans as keyword in context"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "1ef0ee71",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "            Twaelf kragen soo goede als quade\n",
      "      Twaelf slaeplakens. soo goede als quade\n",
      "          divertse bloem, soo goede\n",
      "                soo quade als goede\n",
      "                soo quade als goede\n",
      "                soo quade als goede\n"
     ]
    }
   ],
   "source": [
    "for match in text_helper.find_term_in_context('goede', lines):\n",
    "    print(match['context'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "57e6c515",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         kragen soo goede als quade\n",
      "   slaeplakens. soo goede als quade\n",
      "         bloem, soo goede\n",
      "          quade als goede\n",
      "          quade als goede\n",
      "          quade als goede\n"
     ]
    }
   ],
   "source": [
    "# You can determine the context size\n",
    "for match in text_helper.find_term_in_context('goede', lines, context_size=2):\n",
    "    print(match['context'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "35f38614",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      Twaelf kragen soo goede als quade\n",
      "Twaelf slaeplakens. soo goede als quade\n",
      "    divertse bloem, soo goede\n"
     ]
    }
   ],
   "source": [
    "# You can search for phrases as well as words\n",
    "for match in text_helper.find_term_in_context('soo goede', lines, context_size=2):\n",
    "    print(match['context'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c0f8052",
   "metadata": {},
   "source": [
    "## Finding words similar to a target word\n",
    "\n",
    "With the SkipGram similarity class words from PageXML scans can be indexed on character n-grams with skips, to allow search for words similar to a target word."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8a633c1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('goede', 0.8451542547285165)]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from pagexml.parser import parse_pagexml_file\n",
    "scan = parse_pagexml_file(pagexml_file=filepath)\n",
    "words = scan.get_words()\n",
    "\n",
    "skip_sim = text_helper.SkipgramSimilarity(ngram_length=2, skip_length=1, terms=words)\n",
    "\n",
    "skip_sim.rank_similar('goed')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b62a5287",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}