rag-for-job-apps.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "02a2713c-507b-4576-8780-1643256c4f1f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: langchain_community in /home/chrisflannery/.local/lib/python3.10/site-packages (0.2.12)\n",
      "Requirement already satisfied: tiktoken in /home/chrisflannery/.local/lib/python3.10/site-packages (0.7.0)\n",
      "Requirement already satisfied: langchain-openai in /home/chrisflannery/.local/lib/python3.10/site-packages (0.1.22)\n",
      "Requirement already satisfied: langchainhub in /home/chrisflannery/.local/lib/python3.10/site-packages (0.1.21)\n",
      "Requirement already satisfied: chromadb in /home/chrisflannery/.local/lib/python3.10/site-packages (0.5.5)\n",
      "Requirement already satisfied: langchain in /home/chrisflannery/.local/lib/python3.10/site-packages (0.2.14)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (6.0.2)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (2.0.32)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (3.10.5)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (0.6.7)\n",
      "Requirement already satisfied: langchain-core<0.3.0,>=0.2.30 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (0.2.34)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (0.1.104)\n",
      "Requirement already satisfied: numpy<2,>=1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (1.26.4)\n",
      "Requirement already satisfied: requests<3,>=2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (2.32.3)\n",
      "Requirement already satisfied: tenacity!=8.4.0,<9.0.0,>=8.1.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain_community) (8.5.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /home/chrisflannery/.local/lib/python3.10/site-packages (from tiktoken) (2024.7.24)\n",
      "Requirement already satisfied: openai<2.0.0,>=1.40.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain-openai) (1.42.0)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchainhub) (24.1)\n",
      "Requirement already satisfied: types-requests<3.0.0.0,>=2.31.0.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchainhub) (2.32.0.20240712)\n",
      "Requirement already satisfied: build>=1.0.3 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (1.2.1)\n",
      "Requirement already satisfied: pydantic>=1.9 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (2.8.2)\n",
      "Requirement already satisfied: chroma-hnswlib==0.7.6 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (0.7.6)\n",
      "Requirement already satisfied: fastapi>=0.95.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (0.112.2)\n",
      "Requirement already satisfied: uvicorn>=0.18.3 in /home/chrisflannery/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.30.6)\n",
      "Requirement already satisfied: posthog>=2.4.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (3.5.2)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (4.12.2)\n",
      "Requirement already satisfied: onnxruntime>=1.14.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (1.19.0)\n",
      "Requirement already satisfied: opentelemetry-api>=1.2.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (1.26.0)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (1.26.0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (0.47b0)\n",
      "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (1.26.0)\n",
      "Requirement already satisfied: tokenizers>=0.13.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (0.20.0)\n",
      "Requirement already satisfied: pypika>=0.48.9 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (0.48.9)\n",
      "Requirement already satisfied: tqdm>=4.65.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (4.66.5)\n",
      "Requirement already satisfied: overrides>=7.3.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (7.7.0)\n",
      "Requirement already satisfied: importlib-resources in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (6.4.4)\n",
      "Requirement already satisfied: grpcio>=1.58.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (1.66.0)\n",
      "Requirement already satisfied: bcrypt>=4.0.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (4.2.0)\n",
      "Requirement already satisfied: typer>=0.9.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (0.12.5)\n",
      "Requirement already satisfied: kubernetes>=28.1.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (30.1.0)\n",
      "Requirement already satisfied: mmh3>=4.0.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (4.1.0)\n",
      "Requirement already satisfied: orjson>=3.9.12 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (3.10.7)\n",
      "Requirement already satisfied: httpx>=0.27.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from chromadb) (0.27.0)\n",
      "Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain) (4.0.3)\n",
      "Requirement already satisfied: langchain-text-splitters<0.3.0,>=0.2.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain) (0.2.2)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (2.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (24.2.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/chrisflannery/.local/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.9.4)\n",
      "Requirement already satisfied: pyproject_hooks in /home/chrisflannery/.local/lib/python3.10/site-packages (from build>=1.0.3->chromadb) (1.1.0)\n",
      "Requirement already satisfied: tomli>=1.1.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from build>=1.0.3->chromadb) (2.0.1)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain_community) (3.22.0)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain_community) (0.9.0)\n",
      "Requirement already satisfied: starlette<0.39.0,>=0.37.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb) (0.38.2)\n",
      "Requirement already satisfied: anyio in /home/chrisflannery/.local/lib/python3.10/site-packages (from httpx>=0.27.0->chromadb) (4.4.0)\n",
      "Requirement already satisfied: certifi in /usr/lib/python3/dist-packages (from httpx>=0.27.0->chromadb) (2020.6.20)\n",
      "Requirement already satisfied: httpcore==1.* in /home/chrisflannery/.local/lib/python3.10/site-packages (from httpx>=0.27.0->chromadb) (1.0.5)\n",
      "Requirement already satisfied: idna in /usr/lib/python3/dist-packages (from httpx>=0.27.0->chromadb) (3.3)\n",
      "Requirement already satisfied: sniffio in /home/chrisflannery/.local/lib/python3.10/site-packages (from httpx>=0.27.0->chromadb) (1.3.1)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/chrisflannery/.local/lib/python3.10/site-packages (from httpcore==1.*->httpx>=0.27.0->chromadb) (0.14.0)\n",
      "Requirement already satisfied: six>=1.9.0 in /usr/lib/python3/dist-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in /home/chrisflannery/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.9.0.post0)\n",
      "Requirement already satisfied: google-auth>=1.0.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.34.0)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (1.8.0)\n",
      "Requirement already satisfied: requests-oauthlib in /home/chrisflannery/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.0.0)\n",
      "Requirement already satisfied: oauthlib>=3.2.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
      "Requirement already satisfied: urllib3>=1.24.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.2.2)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /home/chrisflannery/.local/lib/python3.10/site-packages (from langchain-core<0.3.0,>=0.2.30->langchain_community) (1.33)\n",
      "Requirement already satisfied: coloredlogs in /home/chrisflannery/.local/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n",
      "Requirement already satisfied: flatbuffers in /home/chrisflannery/.local/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (24.3.25)\n",
      "Requirement already satisfied: protobuf in /home/chrisflannery/.local/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (4.25.4)\n",
      "Requirement already satisfied: sympy in /home/chrisflannery/.local/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (1.13.2)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.40.0->langchain-openai) (1.7.0)\n",
      "Requirement already satisfied: jiter<1,>=0.4.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from openai<2.0.0,>=1.40.0->langchain-openai) (0.5.0)\n",
      "Requirement already satisfied: deprecated>=1.2.6 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb) (1.2.14)\n",
      "Requirement already satisfied: importlib-metadata<=8.0.0,>=6.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb) (8.0.0)\n",
      "Requirement already satisfied: googleapis-common-protos~=1.52 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.63.2)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.26.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.26.0)\n",
      "Requirement already satisfied: opentelemetry-proto==1.26.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.26.0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.47b0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.47b0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation==0.47b0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.47b0)\n",
      "Requirement already satisfied: opentelemetry-semantic-conventions==0.47b0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.47b0)\n",
      "Requirement already satisfied: opentelemetry-util-http==0.47b0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.47b0)\n",
      "Requirement already satisfied: setuptools>=16.0 in /usr/lib/python3/dist-packages (from opentelemetry-instrumentation==0.47b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (59.6.0)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation==0.47b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.16.0)\n",
      "Requirement already satisfied: asgiref~=3.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-asgi==0.47b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (3.8.1)\n",
      "Requirement already satisfied: monotonic>=1.5 in /home/chrisflannery/.local/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
      "Requirement already satisfied: backoff>=1.10.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from pydantic>=1.9->chromadb) (0.7.0)\n",
      "Requirement already satisfied: pydantic-core==2.20.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from pydantic>=1.9->chromadb) (2.20.1)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from requests<3,>=2->langchain_community) (3.3.2)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /home/chrisflannery/.local/lib/python3.10/site-packages (from SQLAlchemy<3,>=1.4->langchain_community) (3.0.3)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /home/chrisflannery/.local/lib/python3.10/site-packages (from tokenizers>=0.13.2->chromadb) (0.24.6)\n",
      "Requirement already satisfied: click>=8.0.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from typer>=0.9.0->chromadb) (1.5.4)\n",
      "Requirement already satisfied: rich>=10.11.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from typer>=0.9.0->chromadb) (13.7.1)\n",
      "Requirement already satisfied: httptools>=0.5.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.1)\n",
      "Requirement already satisfied: python-dotenv>=0.13 in /home/chrisflannery/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.1)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.20.0)\n",
      "Requirement already satisfied: watchfiles>=0.13 in /home/chrisflannery/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.23.0)\n",
      "Requirement already satisfied: websockets>=10.4 in /home/chrisflannery/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (13.0)\n",
      "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/chrisflannery/.local/lib/python3.10/site-packages (from anyio->httpx>=0.27.0->chromadb) (1.2.2)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.5.0)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /home/chrisflannery/.local/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n",
      "Requirement already satisfied: filelock in /home/chrisflannery/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.15.4)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (2024.6.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /usr/lib/python3/dist-packages (from importlib-metadata<=8.0.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (1.0.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /home/chrisflannery/.local/lib/python3.10/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.3.0,>=0.2.30->langchain_community) (3.0.0)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (2.18.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain_community) (1.0.0)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n",
      "Requirement already satisfied: mpmath<1.4,>=1.1.0 in /home/chrisflannery/.local/lib/python3.10/site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /home/chrisflannery/.local/lib/python3.10/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer>=0.9.0->chromadb) (0.1.2)\n",
      "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /home/chrisflannery/.local/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "pip install langchain_community tiktoken langchain-openai langchainhub chromadb langchain"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f45a36c8-81ed-4601-ba04-ad1e362350dd",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "LANGCHAIN_API_KEY = os.getenv('LANGCHAIN_API_KEY')\n",
    "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY')\n",
    "\n",
    "os.environ['LANGCHAIN_TRACING_V2'] = 'true'\n",
    "os.environ['LANGCHAIN_ENDPOINT'] = 'https://api.smith.langchain.com'\n",
    "os.environ['LANGCHAIN_API_KEY'] = LANGCHAIN_API_KEY\n",
    "os.environ['OPENAI_API_KEY'] = OPENAI_API_KEY\n",
    "os.environ['USER_AGENT'] = \"chris bot (chriswillsflannery@gmail.com)\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f44800f1-9741-48de-a418-a2a81cea8489",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of documents: 44\n",
      "Type: answer, Content: Jupyter notebook with RAG pipeline using these Q&A...\n",
      "Type: answer, Content: Try it out here (Flask app)...\n",
      "Vectorstore created\n"
     ]
    }
   ],
   "source": [
    "from bs4 import BeautifulSoup, SoupStrainer\n",
    "from langchain.schema import Document\n",
    "from langchain import hub\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_community.document_loaders import WebBaseLoader\n",
    "from langchain_community.vectorstores import Chroma\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n",
    "from langchain.prompts import PromptTemplate\n",
    "\n",
    "def parse_html(soup):    \n",
    "    parsed_content = []\n",
    "    for tag in soup.find_all(['h3', 'p']):\n",
    "        if tag.name == 'h3':\n",
    "            parsed_content.append({\"type\": \"question\", \"content\": tag.get_text().strip()})\n",
    "        elif tag.name == 'p':\n",
    "            parsed_content.append({\"type\": \"answer\", \"content\": tag.get_text().strip()})\n",
    "    return parsed_content\n",
    "\n",
    "class CustomWebLoader(WebBaseLoader):\n",
    "    def load(self):\n",
    "        try:\n",
    "            soup = self.scrape()\n",
    "            parsed_content = parse_html(soup)\n",
    "            \n",
    "            docs = []\n",
    "            for item in parsed_content:\n",
    "                metadata = {\"type\": item[\"type\"]}\n",
    "                docs.append(Document(page_content=item[\"content\"], metadata=metadata))\n",
    "            return docs\n",
    "        except Exception as e:\n",
    "            print(f\"An error occurred while loading documents: {e}\")\n",
    "            return []\n",
    "\n",
    "# split pairs \n",
    "def split_qa_pairs(docs):\n",
    "    qa_pairs = []\n",
    "    current_qa = {'question': '', 'answer': ''}\n",
    "    for doc in docs:\n",
    "        if doc.metadata['type'] == 'question':\n",
    "            # If we encounter a new question and we have a previous QA pair, add it to the list\n",
    "            if current_qa['question'] or current_qa['answer']:\n",
    "                qa_pairs.append(Document(\n",
    "                    page_content=f\"Q: {current_qa['question']}\\nA: {current_qa['answer']}\",\n",
    "                    metadata={'type': 'qa_pair'}\n",
    "                ))\n",
    "            # Start a new QA pair\n",
    "            current_qa = {'question': doc.page_content, 'answer': ''}\n",
    "        elif doc.metadata['type'] == 'answer':\n",
    "            # If we encounter an answer without a question, use a placeholder\n",
    "            if not current_qa['question']:\n",
    "                current_qa['question'] = 'Unknown Question'\n",
    "            current_qa['answer'] = doc.page_content\n",
    "    \n",
    "    # Add the last QA pair if it exists\n",
    "    if current_qa['question'] or current_qa['answer']:\n",
    "        qa_pairs.append(Document(\n",
    "            page_content=f\"Q: {current_qa['question']}\\nA: {current_qa['answer']}\",\n",
    "            metadata={'type': 'qa_pair'}\n",
    "        ))\n",
    "    return qa_pairs\n",
    "\n",
    "# Usage remains the same\n",
    "url = \"https://chriswillsflannery.vercel.app/posts/ragExamplesForJobApplication\"\n",
    "loader = CustomWebLoader(url)\n",
    "try:\n",
    "    docs = loader.load()\n",
    "    if not docs:\n",
    "        print(\"No documents were loaded.\")\n",
    "    else:\n",
    "        print(f\"Number of documents: {len(docs)}\")\n",
    "        for doc in docs[:2]:  # Print first two documents as an example\n",
    "            print(f\"Type: {doc.metadata['type']}, Content: {doc.page_content[:100]}...\")\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred while loading documents: {e}\")\n",
    "\n",
    "# Text split if content loaded\n",
    "if docs:\n",
    "    qa_splits = split_qa_pairs(docs)\n",
    "    \n",
    "    try:\n",
    "        vectorstore = Chroma.from_documents(documents=qa_splits, embedding=OpenAIEmbeddings())\n",
    "        print(\"Vectorstore created\")\n",
    "    except Exception as e:\n",
    "        print(f\"Vectorstore creation failed: {e}\")\n",
    "\n",
    "retriever = vectorstore.as_retriever()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "e8f8b2c9-9cd1-4151-9c47-f2f5543a6af4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I approach troubleshooting and debugging with a methodical and collaborative mindset. I believe in first understanding the root cause of the issue by analyzing logs, error messages, and code snippets. I also like to consult with team members or experts in the specific technology if needed, to gain different perspectives and insights. Once I have identified the problem, I work on implementing a solution step by step, testing each change to ensure it resolves the issue without causing any new problems. I am persistent and patient in my approach, always striving to improve my problem-solving skills and learn from each debugging experience.\n"
     ]
    }
   ],
   "source": [
    "# Retrieval and generation\n",
    "prompt_template = PromptTemplate.from_template(\"\"\"\n",
    "You are an assistant helping a job applicant complete job applications. Use the following retrieved context to inform the tone and style of your answers. The context contains examples of the applicant's previous responses to job application questions.\n",
    "\n",
    "Retrieved context:\n",
    "{context}\n",
    "\n",
    "Now, based on the style and content of the above context, please answer the following job application question:\n",
    "\n",
    "Question: {question}\n",
    "\n",
    "Answer:\n",
    "\"\"\")\n",
    "\n",
    "llm = ChatOpenAI(model_name=\"gpt-3.5-turbo\", temperature=0.2)\n",
    "\n",
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(f\"{doc.metadata['type'].capitalize()}: {doc.page_content}\" for doc in docs)\n",
    "\n",
    "\n",
    "rag_chain = (\n",
    "    {\n",
    "        \"context\": retriever | format_docs,\n",
    "        \"question\": RunnablePassthrough()\n",
    "    }\n",
    "    | prompt_template\n",
    "    | llm\n",
    "    | StrOutputParser()\n",
    ")\n",
    "\n",
    "# Usage\n",
    "try:\n",
    "    formatted_docs = format_do\n",
    "    response = rag_chain.invoke(\"How do you approach troubleshooting and debugging issues?\")\n",
    "    print(response)\n",
    "except Exception as e:\n",
    "    print(f\"An error occurred during the RAG chain execution: {e}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b64a8b8a-04ee-4841-a25a-20b733e65c0f",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}