scrapping.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "# Crawl IMdB Website for TOP Grossing Movies and their info from each year"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Notebook Description\n",
        "This notebook is intended for crawling and scraping IMDB website for top grossing movies ( 600 if possible ) for each year between 1920 and 2025.\n",
        "\n",
        "### Dependencies\n",
        "\n",
        "You have to install an ***Microsoft Edge WebDriver***. ( You can change it to any other type of drivers but some minor changes to the code are required.)\n",
        "\n",
        "You can find the ***Microsoft Edge WebDriver*** in this [link](https://developer.microsoft.com/en-us/microsoft-edge/tools/webdriver?form=MA13LH).\n",
        "\n",
        "Any needed libraries will be installed eventually by the notebook.\n",
        "\n",
        "Here are the used libraries:\n",
        "\n",
        "- selenium\n",
        "- beautifulsoup4\n",
        "- pandas"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Scraping Logic\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Installing Dependencies"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "%pip install selenium\n",
        "%pip install bs4\n",
        "%pip install pandas\n",
        "%pip install lxml"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Importing Dependencies."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [],
      "source": [
        "import os\n",
        "import time\n",
        "import logging\n",
        "import pandas as pd\n",
        "from datetime import datetime\n",
        "\n",
        "from bs4 import BeautifulSoup\n",
        "from selenium import webdriver\n",
        "from selenium.webdriver.edge.service import Service\n",
        "from selenium.webdriver.common.by import By\n",
        "from selenium.webdriver.support.ui import WebDriverWait\n",
        "from selenium.webdriver.support import expected_conditions as EC"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### setting up the directory structure and logging configuration."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "def setup_directories(year):\n",
        "    \"\"\"\n",
        "    Creates the necessary directory structure for a specific year\n",
        "    \"\"\"\n",
        "\n",
        "    data_dir = f\"Data/{year}\"\n",
        "    os.makedirs(data_dir, exist_ok=True)\n",
        "    \n",
        "    logs_dir = f\"Logs/{year}\"\n",
        "    os.makedirs(logs_dir, exist_ok=True)\n",
        "    \n",
        "    return data_dir, logs_dir\n",
        "\n",
        "def setup_logging(year):\n",
        "    \"\"\"\n",
        "    Configures logging for errors and results for a specific year\n",
        "    \"\"\"\n",
        "    \n",
        "    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')\n",
        "    \n",
        "    # Error Logger\n",
        "    error_logger = logging.getLogger(f\"error_logger_{year}\")\n",
        "    error_logger.setLevel(logging.ERROR)\n",
        "    \n",
        "    # file handler for error logs\n",
        "    error_handler = logging.FileHandler(f\"Logs/{year}/errors.txt\")\n",
        "    error_handler.setLevel(logging.ERROR)\n",
        "    error_handler.setFormatter(formatter)\n",
        "    \n",
        "    error_logger.addHandler(error_handler)\n",
        "    \n",
        "    # Results Logger\n",
        "    results_logger = logging.getLogger(f\"results_logger_{year}\")\n",
        "    results_logger.setLevel(logging.INFO)\n",
        "    \n",
        "    # file handler for results logs\n",
        "    results_handler = logging.FileHandler(f\"Logs/{year}/results.txt\")\n",
        "    results_handler.setLevel(logging.INFO)\n",
        "    results_handler.setFormatter(formatter)\n",
        "    \n",
        "    results_logger.addHandler(results_handler)\n",
        "    \n",
        "    return error_logger, results_logger"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Utility Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Link Extraction Function"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [],
      "source": [
        "def extract_links(year, error_logger, results_logger):\n",
        "    \"\"\"\n",
        "    Extracts movie links from IMDB for a specific year\n",
        "\n",
        "    Args:\n",
        "        year (int): The year to extract data for\n",
        "        error_logger: Logger for error messages\n",
        "        results_logger: Logger for results statistics\n",
        "\n",
        "    Returns:\n",
        "        pandas.DataFrame: DataFrame containing extracted movie links and basic information\n",
        "    \"\"\"\n",
        "    start_time = time.time()\n",
        "\n",
        "    results_logger.info(f\"Starting link extraction for year {year}\")\n",
        "\n",
        "    url = f\"https://www.imdb.com/search/title/?title_type=feature&release_date={year}-01-01,{year}-12-31&count=50&sort=boxoffice_gross_us,desc\"\n",
        "\n",
        "    driver_path = \"edgedriver.exe\"\n",
        "    options = webdriver.EdgeOptions()\n",
        "    options.add_argument(\"--lang=en-US\")\n",
        "    options.add_argument(\"--disable-blink-features=AutomationControlled\")\n",
        "    options.add_argument(\n",
        "        \"--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0\"\n",
        "    )\n",
        "    options.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n",
        "    options.add_experimental_option(\"useAutomationExtension\", False)\n",
        "\n",
        "    options.add_argument(\"--disable-gpu\")\n",
        "    options.add_argument(\"--no-sandbox\")\n",
        "    options.add_argument(\"--disable-dev-shm-usage\")\n",
        "    options.add_argument(\"--disable-extensions\")\n",
        "    options.add_argument(\"--disable-infobars\")\n",
        "\n",
        "    service = Service(executable_path=driver_path)\n",
        "    driver = webdriver.Edge(service=service, options=options)\n",
        "    driver.set_window_size(800, 600)\n",
        "\n",
        "    # Initialize data container\n",
        "    films_data = []\n",
        "    errors_count = 0\n",
        "\n",
        "    try:\n",
        "        driver.get(url)\n",
        "        WebDriverWait(driver, 15).until(\n",
        "            EC.presence_of_element_located((By.CSS_SELECTOR, \"ul.ipc-metadata-list\"))\n",
        "        )\n",
        "\n",
        "        # Load more results until we reach 600 or there are no more to load\n",
        "        loaded_data = 50\n",
        "        while loaded_data < 600:\n",
        "            try:\n",
        "                # Find and click the \"Load more\" button\n",
        "                load_more_button = WebDriverWait(driver, 10).until(\n",
        "                    EC.presence_of_element_located(\n",
        "                        (\n",
        "                            By.XPATH,\n",
        "                            \"//button[contains(@class, 'ipc-btn') and .//span[contains(text(), '50 more')]]\",\n",
        "                        )\n",
        "                    )\n",
        "                )\n",
        "\n",
        "                driver.execute_script(\n",
        "                    \"arguments[0].scrollIntoView(true);\", load_more_button\n",
        "                )\n",
        "\n",
        "                driver.execute_script(\"arguments[0].click();\", load_more_button)\n",
        "\n",
        "                time.sleep(5)\n",
        "                loaded_data += 50\n",
        "\n",
        "            except Exception as e:\n",
        "                # Log the error and break the loop\n",
        "                error_message = (\n",
        "                    f\"No more 'Load More' button found or an error occurred: {e}\"\n",
        "                )\n",
        "                error_logger.error(error_message)\n",
        "                break\n",
        "\n",
        "        results_logger.info(f\"Loaded {loaded_data} items before stopping\")\n",
        "\n",
        "        # Extract data from the page\n",
        "        html = driver.page_source\n",
        "        soup = BeautifulSoup(html, \"lxml\")\n",
        "        del html\n",
        "        \n",
        "        # Find the list of films\n",
        "        films = soup.select(\"ul.ipc-metadata-list\")\n",
        "        if films and len(films) > 0:\n",
        "            films = films[0]\n",
        "            # Check what you actually got\n",
        "            results_logger.info(\n",
        "                f\"Found film list container: {films.name}, classes: {films.get('class')}\"\n",
        "            )\n",
        "\n",
        "        if films:\n",
        "            # Process each film in the list\n",
        "            for film in films.find_all(\"li\", class_=\"ipc-metadata-list-summary-item\"):\n",
        "                try:\n",
        "                    # Extract title\n",
        "                    title = (\n",
        "                        film.find(\"h3\", class_=\"ipc-title__text\").text\n",
        "                        if film.find(\"h3\", class_=\"ipc-title__text\")\n",
        "                        else None\n",
        "                    )\n",
        "\n",
        "                    # Extract metadata\n",
        "                    metadata_div = film.find(\"div\", class_=\"dli-title-metadata\")\n",
        "\n",
        "                    # Extract year, duration, and MPA rating\n",
        "                    year_data = (\n",
        "                        metadata_div.find_all(\"span\")[0].text\n",
        "                        if metadata_div and len(metadata_div.find_all(\"span\")) > 0\n",
        "                        else None\n",
        "                    )\n",
        "                    duration = (\n",
        "                        metadata_div.find_all(\"span\")[1].text\n",
        "                        if metadata_div and len(metadata_div.find_all(\"span\")) > 1\n",
        "                        else None\n",
        "                    )\n",
        "                    mpa = (\n",
        "                        metadata_div.find_all(\"span\")[2].text\n",
        "                        if metadata_div and len(metadata_div.find_all(\"span\")) > 2\n",
        "                        else None\n",
        "                    )\n",
        "\n",
        "                    # Extract rating\n",
        "                    rating_info = film.find(\"span\", class_=\"ipc-rating-star--rating\")\n",
        "                    rating = rating_info.text if rating_info else None\n",
        "\n",
        "                    # Extract movie link\n",
        "                    link_tag = film.find(\"a\", class_=\"ipc-lockup-overlay ipc-focusable\")\n",
        "                    movie_link = (\n",
        "                        f\"https://www.imdb.com{link_tag['href']}\" if link_tag else None\n",
        "                    )\n",
        "\n",
        "                    # Extract vote count\n",
        "                    vote_count_info = film.find(\n",
        "                        \"span\", class_=\"ipc-rating-star--voteCount\"\n",
        "                    )\n",
        "                    vote_count = (\n",
        "                        vote_count_info.text.strip().replace(\"\\xa0\", \"\")[1:-1]\n",
        "                        if vote_count_info\n",
        "                        else None\n",
        "                    )\n",
        "\n",
        "                    # Extract meta score\n",
        "                    meta_score_info = film.find(\"span\", class_=\"metacritic-score-box\")\n",
        "                    meta_score = meta_score_info.text if meta_score_info else None\n",
        "\n",
        "                    # Extract description\n",
        "                    description_div = film.find(\n",
        "                        \"div\", class_=\"ipc-html-content-inner-div\"\n",
        "                    )\n",
        "                    description = (\n",
        "                        description_div.text.strip() if description_div else None\n",
        "                    )\n",
        "\n",
        "                    # Add data to the list\n",
        "                    films_data.append(\n",
        "                        {\n",
        "                            \"Title\": title,\n",
        "                            \"Year\": year_data,\n",
        "                            \"Duration\": duration,\n",
        "                            \"MPA\": mpa,\n",
        "                            \"Rating\": rating,\n",
        "                            \"Votes\": vote_count,\n",
        "                            \"méta_score\": meta_score,\n",
        "                            \"description\": description,\n",
        "                            \"Movie Link\": movie_link,\n",
        "                        }\n",
        "                    )\n",
        "\n",
        "                except Exception as e:\n",
        "                    # Log individual film extraction errors\n",
        "                    error_logger.error(f\"Error extracting data for a film: {e}\")\n",
        "                    errors_count += 1\n",
        "        else:\n",
        "            error_logger.error(\"Could not find the film list element on the page\")\n",
        "\n",
        "    except Exception as e:\n",
        "        # Log any major errors during the extraction process\n",
        "        error_logger.error(f\"Error during link extraction: {e}\")\n",
        "        errors_count += 1\n",
        "\n",
        "    finally:\n",
        "        driver.quit()\n",
        "\n",
        "    df = pd.DataFrame(films_data)\n",
        "\n",
        "    elapsed_time = time.time() - start_time\n",
        "\n",
        "    results_logger.info(f\"Link extraction completed for year {year}\")\n",
        "    results_logger.info(f\"Total movies extracted: {len(films_data)}\")\n",
        "    results_logger.info(f\"Errors encountered: {errors_count}\")\n",
        "    results_logger.info(f\"Elapsed time: {elapsed_time:.2f} seconds\")\n",
        "\n",
        "    return df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Advanced Data Extraction Function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "def extract_advanced_data(year, links_df, error_logger, results_logger):\n",
        "    \"\"\"\n",
        "    Extracts detailed movie data from IMDB for a specific year\n",
        "\n",
        "    Args:\n",
        "        year (int): The year to extract data for\n",
        "        links_df (pandas.DataFrame): DataFrame containing movie links\n",
        "        error_logger: Logger for error messages\n",
        "        results_logger: Logger for results statistics\n",
        "\n",
        "    Returns:\n",
        "        pandas.DataFrame: DataFrame containing detailed movie information\n",
        "    \"\"\"\n",
        "    start_time = time.time()\n",
        "    results_logger.info(f\"Starting advanced data extraction for year {year}\")\n",
        "\n",
        "    all_movie_data = []\n",
        "    errors_count = 0\n",
        "\n",
        "    # Set up WebDriver\n",
        "    driver_path = \"edgedriver.exe\"\n",
        "    options = webdriver.EdgeOptions()\n",
        "    options.add_argument(\"--lang=en-US\")\n",
        "    options.add_argument(\"--disable-blink-features=AutomationControlled\")\n",
        "    options.add_argument(\n",
        "        \"--user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36 Edg/121.0.0.0\"\n",
        "    )\n",
        "    options.add_experimental_option(\"excludeSwitches\", [\"enable-automation\"])\n",
        "    options.add_experimental_option(\"useAutomationExtension\", False)\n",
        "\n",
        "    options.add_argument(\"--disable-gpu\")\n",
        "    options.add_argument(\"--no-sandbox\")\n",
        "    options.add_argument(\"--disable-dev-shm-usage\")\n",
        "    options.add_argument(\"--disable-extensions\")\n",
        "    options.add_argument(\"--disable-infobars\")\n",
        "\n",
        "    service = Service(executable_path=driver_path)\n",
        "    driver = webdriver.Edge(service=service, options=options)\n",
        "    driver.set_window_size(800, 600)\n",
        "\n",
        "    # Process each movie URL\n",
        "    for url in list(links_df[\"Movie Link\"]):\n",
        "        try:\n",
        "            driver.get(url)\n",
        "            driver.execute_script(\"window.scrollTo(0, document.body.scrollHeight);\")\n",
        "            WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.TAG_NAME, \"footer\")))\n",
        "            time.sleep(0.05)\n",
        "            html = driver.page_source\n",
        "            soup = BeautifulSoup(html, \"lxml\")\n",
        "            del html\n",
        "            \n",
        "            # Initialize variables for data extraction\n",
        "            budget_text = None\n",
        "            opening_weekend_text = None\n",
        "            grossWorldWide_text = None\n",
        "            gross_US_Canada = None\n",
        "            release_date_text = None\n",
        "            list_countries_origin = None\n",
        "            filmingLocation_texts = None\n",
        "            productionCompany_text = None\n",
        "            list_stars = None\n",
        "            awards_content = None\n",
        "            writers_text = None\n",
        "            directors_text = None\n",
        "            genres_text = []\n",
        "            languages_list = []\n",
        "\n",
        "            # Extract budget information\n",
        "            try:\n",
        "                budget = soup.find(\"li\", {\"data-testid\": \"title-boxoffice-budget\"})\n",
        "                if budget:\n",
        "                    budget_text = (\n",
        "                        budget.find(\n",
        "                            \"span\",\n",
        "                            {\"class\": \"ipc-metadata-list-item__list-content-item\"},\n",
        "                        )\n",
        "                        .text.replace(\"\\u202f\", \",\")\n",
        "                        .replace(\"\\xa0\", \"\")\n",
        "                    )\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting budget for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract opening weekend information\n",
        "            try:\n",
        "                opening_weekend = soup.find(\n",
        "                    \"li\", {\"data-testid\": \"title-boxoffice-openingweekenddomestic\"}\n",
        "                )\n",
        "                if opening_weekend:\n",
        "                    opening_weekend_text = (\n",
        "                        opening_weekend.find_all(\n",
        "                            \"span\",\n",
        "                            {\"class\": \"ipc-metadata-list-item__list-content-item\"},\n",
        "                        )[0]\n",
        "                        .text.replace(\"\\u202f\", \",\")\n",
        "                        .replace(\"\\xa0\", \"\")\n",
        "                    )\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting opening weekend for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract worldwide gross information\n",
        "            try:\n",
        "                gross_worldwide = soup.find(\n",
        "                    \"li\", {\"data-testid\": \"title-boxoffice-cumulativeworldwidegross\"}\n",
        "                )\n",
        "                if gross_worldwide:\n",
        "                    grossWorldWide_text = (\n",
        "                        gross_worldwide.find(\n",
        "                            \"span\",\n",
        "                            {\"class\": \"ipc-metadata-list-item__list-content-item\"},\n",
        "                        )\n",
        "                        .text.replace(\"\\u202f\", \",\")\n",
        "                        .replace(\"\\xa0\", \"\")\n",
        "                    )\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting worldwide gross for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract US/Canada gross information\n",
        "            try:\n",
        "                gross_US_Canada_section = soup.find(\n",
        "                    \"li\", {\"data-testid\": \"title-boxoffice-grossdomestic\"}\n",
        "                )\n",
        "                if gross_US_Canada_section:\n",
        "                    gross_US_Canada = (\n",
        "                        gross_US_Canada_section.find(\n",
        "                            \"span\",\n",
        "                            {\"class\": \"ipc-metadata-list-item__list-content-item\"},\n",
        "                        )\n",
        "                        .text.replace(\"\\u202f\", \",\")\n",
        "                        .replace(\"\\xa0\", \"\")\n",
        "                    )\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting US/Canada gross for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract countries of origin\n",
        "            try:\n",
        "                countries_origin = soup.find(\n",
        "                    \"li\", {\"data-testid\": \"title-details-origin\"}\n",
        "                )\n",
        "                if countries_origin:\n",
        "                    countries_list = countries_origin.find_all(\n",
        "                        \"a\", class_=\"ipc-metadata-list-item__list-content-item\"\n",
        "                    )\n",
        "                    list_countries_origin = [\n",
        "                        country.get_text() for country in countries_list\n",
        "                    ]\n",
        "                else:\n",
        "                    list_countries_origin = None\n",
        "            except Exception as e:\n",
        "                error_logger.error(\n",
        "                    f\"Error extracting countries of origin for {url}: {e}\"\n",
        "                )\n",
        "                list_countries_origin = None\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract genres\n",
        "            try:\n",
        "                interests_section = soup.find(\"div\", {\"data-testid\": \"interests\"})\n",
        "                if interests_section:\n",
        "                    genres = interests_section.find_all(\"span\", class_=\"ipc-chip__text\")\n",
        "                    genres_text = [genre.get_text() for genre in genres]\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting genres for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract languages\n",
        "            try:\n",
        "                languages_section = soup.find(\n",
        "                    \"li\", {\"data-testid\": \"title-details-languages\"}\n",
        "                )\n",
        "                if languages_section:\n",
        "                    languages = languages_section.find_all(\n",
        "                        \"a\",\n",
        "                        class_=\"ipc-metadata-list-item__list-content-item\",\n",
        "                    )\n",
        "                    languages_list = [lang.get_text() for lang in languages]\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting languages for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract awards information\n",
        "            try:\n",
        "                awards_div = soup.find(\"div\", {\"data-testid\": \"awards\"})\n",
        "                if awards_div:\n",
        "                    text = awards_div.find(\n",
        "                        \"a\", class_=\"ipc-metadata-list-item__label\"\n",
        "                    ).get_text()\n",
        "                    if not text:\n",
        "                        text = \"\"\n",
        "                    else:\n",
        "                        text += \", \"\n",
        "                    awards_content = (\n",
        "                        text\n",
        "                        + awards_div.find(\n",
        "                            \"span\", class_=\"ipc-metadata-list-item__list-content-item\"\n",
        "                        ).get_text()\n",
        "                    )\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting awards for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract filming locations\n",
        "            try:\n",
        "                filming_location_section = soup.find(\n",
        "                    \"li\", {\"data-testid\": \"title-details-filminglocations\"}\n",
        "                )\n",
        "                if filming_location_section:\n",
        "                    all_filming_locations = filming_location_section.find_all(\n",
        "                        \"li\", {\"class\": \"ipc-inline-list__item\"}\n",
        "                    )\n",
        "                    filmingLocation_texts = [\n",
        "                        (\n",
        "                            (\n",
        "                                filming_location_li.find(\"a\").get_text()\n",
        "                                + \" \"\n",
        "                                + filming_location_li.find(\"span\").get_text()\n",
        "                            )\n",
        "                            if filming_location_li.find(\"a\")\n",
        "                            and filming_location_li.find(\"span\")\n",
        "                            else filming_location_li.find(\"a\").get_text()\n",
        "                        )\n",
        "                        for filming_location_li in all_filming_locations\n",
        "                    ]\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting filming locations for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract writers and directors\n",
        "            principal_credit = soup.find_all(\"li\", {\"class\": \"ipc-metadata-list__item\"})\n",
        "\n",
        "            try:\n",
        "                writers_div = principal_credit[1]\n",
        "                if writers_div:\n",
        "                    writers_links = writers_div.find_all(\n",
        "                        \"a\", {\"class\": \"ipc-metadata-list-item__list-content-item\"}\n",
        "                    )\n",
        "                    writers_text = [writer.get_text() for writer in writers_links]\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting writers for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "            try:\n",
        "                director_div = principal_credit[0]\n",
        "                if director_div:\n",
        "                    directors_links = director_div.find_all(\n",
        "                        \"a\", {\"class\": \"ipc-metadata-list-item__list-content-item\"}\n",
        "                    )\n",
        "                    directors_text = [\n",
        "                        director.get_text() for director in directors_links\n",
        "                    ]\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting director for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract production companies\n",
        "            try:\n",
        "                production_companies_section = soup.find(\n",
        "                    \"li\", {\"data-testid\": \"title-details-companies\"}\n",
        "                )\n",
        "                if production_companies_section:\n",
        "                    companies = production_companies_section.find_all(\n",
        "                        \"a\", {\"class\": \"ipc-metadata-list-item__list-content-item\"}\n",
        "                    )\n",
        "                    productionCompany_text = [company.text for company in companies]\n",
        "            except Exception as e:\n",
        "                error_logger.error(\n",
        "                    f\"Error extracting production companies for {url}: {e}\"\n",
        "                )\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract release date\n",
        "            try:\n",
        "                release_date_section = soup.find(\n",
        "                    \"li\", {\"data-testid\": \"title-details-releasedate\"}\n",
        "                )\n",
        "                if release_date_section:\n",
        "                    release_date_text = release_date_section.find(\n",
        "                        \"a\", {\"class\": \"ipc-metadata-list-item__list-content-item\"}\n",
        "                    ).text.split(\" (\")[0]\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting release date for {url}: {e}\")\n",
        "                errors_count += 1\n",
        "\n",
        "            # Extract stars/actors\n",
        "            try:\n",
        "                actors_grid = soup.find(\n",
        "                    \"div\",\n",
        "                    class_=\"ipc-sub-grid ipc-sub-grid--page-span-2 ipc-sub-grid--wraps-at-above-l ipc-shoveler__grid\",\n",
        "                )\n",
        "                if actors_grid:\n",
        "                    actor_divs = actors_grid.find_all(\n",
        "                        \"div\", {\"data-testid\": \"title-cast-item\"}, limit=10\n",
        "                    )\n",
        "                    list_stars = [\n",
        "                        actor_div.find(\n",
        "                            \"a\", {\"data-testid\": \"title-cast-item__actor\"}\n",
        "                        ).get_text()\n",
        "                        for actor_div in actor_divs\n",
        "                        if actor_div.find(\n",
        "                            \"a\", {\"data-testid\": \"title-cast-item__actor\"}\n",
        "                        )\n",
        "                    ]\n",
        "                else:\n",
        "                    list_stars = None\n",
        "            except Exception as e:\n",
        "                error_logger.error(f\"Error extracting stars for {url}: {e}\")\n",
        "                list_stars = None\n",
        "                errors_count += 1\n",
        "\n",
        "            # Add all extracted data to the list\n",
        "            all_movie_data.append(\n",
        "                {\n",
        "                    \"link\": url,\n",
        "                    \"writers\": writers_text,\n",
        "                    \"directors\": directors_text,\n",
        "                    \"stars\": list_stars,\n",
        "                    \"budget\": budget_text,\n",
        "                    \"opening_weekend_Gross\": opening_weekend_text,\n",
        "                    \"grossWorldWWide\": grossWorldWide_text,\n",
        "                    \"gross_US_Canada\": gross_US_Canada,\n",
        "                    \"release_date\": release_date_text,\n",
        "                    \"countries_origin\": list_countries_origin,\n",
        "                    \"filming_locations\": filmingLocation_texts,\n",
        "                    \"production_company\": productionCompany_text,\n",
        "                    \"awards_content\": awards_content,\n",
        "                    \"genres\": genres_text,\n",
        "                    \"Languages\": languages_list,\n",
        "                }\n",
        "            )\n",
        "\n",
        "        except Exception as e:\n",
        "            # Log any major errors during the extraction process for this URL\n",
        "            error_logger.error(f\"Error processing URL {url}: {e}\")\n",
        "            errors_count += 1\n",
        "\n",
        "    driver.quit()\n",
        "\n",
        "    movies_data = pd.DataFrame(all_movie_data)\n",
        "    elapsed_time = time.time() - start_time\n",
        "\n",
        "    results_logger.info(f\"Advanced data extraction completed for year {year}\")\n",
        "    results_logger.info(f\"Total movies processed: {len(all_movie_data)}\")\n",
        "    results_logger.info(f\"Errors encountered: {errors_count}\")\n",
        "    results_logger.info(f\"Elapsed time: {elapsed_time:.2f} seconds\")\n",
        "\n",
        "    return movies_data"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Merge Data Function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [],
      "source": [
        "def merge_data(year, data_dir, error_logger, results_logger):\n",
        "    \"\"\"\n",
        "    Merges basic and advanced movie data for a specific year\n",
        "\n",
        "    Args:\n",
        "        year (int): The year to merge data for\n",
        "        data_dir (str): Directory path for data files\n",
        "        error_logger: Logger for error messages\n",
        "        results_logger: Logger for results statistics\n",
        "\n",
        "    Returns:\n",
        "        pandas.DataFrame: Merged DataFrame with all movie information\n",
        "    \"\"\"\n",
        "    start_time = time.time()\n",
        "    results_logger.info(f\"Starting data merging for year {year}\")\n",
        "\n",
        "    try:\n",
        "\n",
        "        advanced_file = f\"{data_dir}/advanced_movies_details_{year}.csv\"\n",
        "        basic_file = f\"{data_dir}/imdb_movies_{year}.csv\"\n",
        "\n",
        "        movies_data = pd.read_csv(advanced_file)\n",
        "        df = pd.read_csv(basic_file)\n",
        "\n",
        "        # Rename the link column to match between datasets\n",
        "        movies_data.rename(columns={\"link\": \"Movie Link\"}, inplace=True)\n",
        "\n",
        "        # Merge the datasets\n",
        "        merged_data = pd.merge(df, movies_data, how=\"inner\", on=\"Movie Link\")\n",
        "        elapsed_time = time.time() - start_time\n",
        "\n",
        "        results_logger.info(f\"Basic data rows: {len(df)}\")\n",
        "        results_logger.info(f\"Advanced data rows: {len(movies_data)}\")\n",
        "        results_logger.info(f\"Merged data rows: {len(merged_data)}\")\n",
        "        results_logger.info(f\"Data merging completed in {elapsed_time:.2f} seconds\")\n",
        "\n",
        "        return merged_data\n",
        "\n",
        "    except Exception as e:\n",
        "        # Log any errors during the merging process\n",
        "        error_logger.error(f\"Error merging data for year {year}: {e}\")\n",
        "        results_logger.info(f\"Data merging failed for year {year}\")\n",
        "        return None"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Main Processing Function."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {},
      "outputs": [],
      "source": [
        "def process_year(year):\n",
        "    \"\"\"\n",
        "    Processes IMDB data for a specific year:\n",
        "    1. Extracts basic movie links and information\n",
        "    2. Extracts advanced movie details\n",
        "    3. Merges the datasets\n",
        "    4. Saves all files to appropriate directories\n",
        "\n",
        "    Args:\n",
        "        year (int): The year to process\n",
        "    \"\"\"\n",
        "    print(f\"Starting processing for year {year}\")\n",
        "    start_time = time.time()\n",
        "\n",
        "    # Setup directories and logging\n",
        "    data_dir, logs_dir = setup_directories(year)\n",
        "    error_logger, results_logger = setup_logging(year)\n",
        "\n",
        "    try:\n",
        "        results_logger.info(\n",
        "            f\"===== Starting IMDB data processing for year {year} =====\"\n",
        "        )\n",
        "        results_logger.info(\n",
        "            f\"Start time: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\"\n",
        "        )\n",
        "\n",
        "        # Step 1: Extract basic movie links and information\n",
        "        print(f\"Extracting basic movie information for {year}...\")\n",
        "        links_df = extract_links(year, error_logger, results_logger)\n",
        "        basic_file = f\"{data_dir}/imdb_movies_{year}.csv\"\n",
        "        links_df.to_csv(basic_file, index=False)\n",
        "        results_logger.info(f\"Saved basic movie information to {basic_file}\")\n",
        "        print(f\"Basic movie information saved to {basic_file}\")\n",
        "\n",
        "        # Step 2: Extract advanced movie details\n",
        "        print(f\"Extracting advanced movie details for {year}...\")\n",
        "        advanced_df = extract_advanced_data(\n",
        "            year, links_df, error_logger, results_logger\n",
        "        )\n",
        "        advanced_file = f\"{data_dir}/advanced_movies_details_{year}.csv\"\n",
        "        advanced_df.to_csv(advanced_file, index=False)\n",
        "        results_logger.info(f\"Saved advanced movie details to {advanced_file}\")\n",
        "        print(f\"Advanced movie details saved to {advanced_file}\")\n",
        "\n",
        "        # Step 3: Merge the datasets\n",
        "        print(f\"Merging movie data for {year}...\")\n",
        "        merged_df = merge_data(year, data_dir, error_logger, results_logger)\n",
        "\n",
        "        if merged_df is not None:\n",
        "            merged_file = f\"{data_dir}/merged_movies_data_{year}.csv\"\n",
        "            merged_df.to_csv(merged_file, index=False)\n",
        "            results_logger.info(f\"Saved merged movie data to {merged_file}\")\n",
        "            print(f\"Merged movie data saved to {merged_file}\")\n",
        "        else:\n",
        "            print(f\"Error: Failed to merge data for {year}\")\n",
        "\n",
        "        total_elapsed_time = time.time() - start_time\n",
        "        results_logger.info(\n",
        "            f\"Total processing time for year {year}: {total_elapsed_time:.2f} seconds\"\n",
        "        )\n",
        "        results_logger.info(\n",
        "            f\"===== Completed IMDB data processing for year {year} =====\"\n",
        "        )\n",
        "\n",
        "        print(\n",
        "            f\"Processing completed for year {year}. Total time: {total_elapsed_time:.2f} seconds\"\n",
        "        )\n",
        "\n",
        "    except Exception as e:\n",
        "        # Log any major errors during the overall process\n",
        "        error_logger.error(f\"Critical error during processing for year {year}: {e}\")\n",
        "        results_logger.info(f\"Processing failed for year {year}\")\n",
        "        print(f\"Error: Processing failed for year {year}: {e}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "start_year = 1920\n",
        "end_year = 2025\n",
        "\n",
        "for year in range(start_year, end_year + 1):\n",
        "    print(f\"\\n{'='*50}\\nProcessing year {year}\\n{'='*50}\")\n",
        "    process_year(year)\n",
        "    \n",
        "    # avoid potential rate limiting\n",
        "    if year < end_year:\n",
        "        print(f\"Waiting 8 seconds before processing the next year...\")\n",
        "        time.sleep(8)\n",
        "        \n",
        "print(\"\\nAll years processed successfully!\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.13.2"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}