Fine-Tuning GPT-2 on PubMed Articles.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "379c2d60-ae4f-4a20-960e-a04b55999145",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Dec 2023\n",
    "# Author: SBN\n",
    "# inspired by https://www.kaggle.com/code/tahmidmir/fine-tuning-gpt-2-on-skin-cancer-articles"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "980d2024-9a01-437c-bb34-72b42f031332",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "276e462a-921b-4fae-b60b-e962dcdda71c",
   "metadata": {},
   "outputs": [],
   "source": [
    "from Bio import Entrez\n",
    "import pandas as pd\n",
    "import time\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "81ee857a-a6bd-4161-8b83-9ecd225f6232",
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment for insalling:\n",
    "# !pip install biopython\n",
    "# !pip install pytz"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8ae2bc4b-79b5-477e-952f-66630b5aeb12",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dbcd3f83-b889-4aeb-a666-b15a90cee910",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2b2a179e-6c16-4445-a36c-fff6ebb2dc4e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9be90713-a04e-4c83-abd5-b443b0197eb2",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "# NCBI API settings\n",
    "Entrez.email = \"mirtahmid@gmail.com\"  # Required by NCBI to use the API\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cba15a04-3705-4b8e-8230-afedd0480f4b",
   "metadata": {},
   "source": [
    "<a id=\"1\"></a>\n",
    "# <div style=\"text-align:center; border-radius:3px 50px; padding:7px; color:white; margin:0; font-size:100%; font-family:Pacifico; background-color:#a0b1b0; \"><b>⭐ Fetching data⭐</b></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c2d1d563-b353-4642-9d35-e882ea655d1e",
   "metadata": {},
   "outputs": [],
   "source": [
    "num_article = 1000\n",
    "handle = Entrez.esearch(db=\"pubmed\", term=\"skin cancer\", retmax=num_article)\n",
    "record = Entrez.read(handle)\n",
    "ids = record['IdList']\n",
    "handle.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a5ffba23-9a11-4c91-bda0-cd550ea343f6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "0ce52ae6-edf3-4cad-a66a-1536d5a850b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['39303343', '39303804']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ids[:-3:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5190a585-c25c-4360-8c1b-d663fcb4fa87",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "20961474-9aa1-4944-bb2c-1a77d2d9d510",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "26b5b9c4-5673-4df3-8a9b-4e0c21b7d35f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fetch articles with the specified PubMed IDs\n",
    "handle = Entrez.efetch(db=\"pubmed\", id=\",\".join(ids), rettype=\"xml\", retmode=\"text\")\n",
    "records = Entrez.read(handle)\n",
    "handle.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "ef647bd0-116c-40d9-bc95-dde2c6691b71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['PubmedBookArticle', 'PubmedArticle'])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records.keys()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a28b103-f888-4934-bddd-0c84a1bc7e0e",
   "metadata": {},
   "source": [
    "<a id=\"1\"></a>\n",
    "# <div style=\"text-align:center; border-radius:3px 50px; padding:7px; color:white; margin:0; font-size:100%; font-family:Pacifico; background-color:#a0b1b0; \"><b>⭐ Observe the data parts⭐</b></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "1fed62be-9c15-456b-86ec-22f9b43e4691",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'MedlineCitation': DictElement({'OtherID': [], 'CitationSubset': ['IM'], 'KeywordList': [ListElement([StringElement('accelerated failure time model', attributes={'MajorTopicYN': 'N'}), StringElement('cured model', attributes={'MajorTopicYN': 'N'}), StringElement('partly interval censoring', attributes={'MajorTopicYN': 'N'}), StringElement('penalized likelihood', attributes={'MajorTopicYN': 'N'}), StringElement('semiparametric estimation', attributes={'MajorTopicYN': 'N'})], attributes={'Owner': 'NOTNLM'})], 'OtherAbstract': [], 'InvestigatorList': [], 'GeneralNote': [], 'SpaceFlightMission': [], 'PMID': StringElement('39508209', attributes={'Version': '1'}), 'DateCompleted': {'Year': '2024', 'Month': '11', 'Day': '07'}, 'DateRevised': {'Year': '2024', 'Month': '11', 'Day': '07'}, 'Article': DictElement({'ELocationID': [StringElement('10.1002/bimj.202300203', attributes={'EIdType': 'doi', 'ValidYN': 'Y'})], 'ArticleDate': [], 'Language': ['eng'], 'Journal': {'ISSN': StringElement('1521-4036', attributes={'IssnType': 'Electronic'}), 'JournalIssue': DictElement({'Volume': '66', 'Issue': '8', 'PubDate': {'Year': '2024', 'Month': 'Dec'}}, attributes={'CitedMedium': 'Internet'}), 'Title': 'Biometrical journal. Biometrische Zeitschrift', 'ISOAbbreviation': 'Biom J'}, 'ArticleTitle': 'Mixture Cure Semiparametric Accelerated Failure Time Models With Partly Interval-Censored Data.', 'Pagination': {'StartPage': 'e202300203', 'MedlinePgn': 'e202300203'}, 'Abstract': {'AbstractText': ['In practical survival analysis, the situation of no event for a patient can arise even after a long period of waiting time, which means a portion of the population may never experience the event of interest. Under this circumstance, one remedy is to adopt a mixture cure Cox model to analyze the survival data. However, if there clearly exhibits an acceleration (or deceleration) factor among their survival times, then an accelerated failure time (AFT) model will be preferred, leading to a mixture cure AFT model. In this paper, we consider a penalized likelihood method to estimate the mixture cure semiparametric AFT models, where the unknown baseline hazard is approximated using Gaussian basis functions. We allow partly interval-censored survival data which can include event times and left-, right-, and interval-censoring times. The penalty function helps to achieve a smooth estimate of the baseline hazard function. We will also provide asymptotic properties to the estimates so that inferences can be made on regression parameters and hazard-related quantities. Simulation studies are conducted to evaluate the model performance, which includes a comparative study with an existing method from the smcure R package. The results show that our proposed penalized likelihood method has acceptable performance in general and produces less bias when faced with the identifiability issue compared to smcure. To illustrate the application of our method, a real case study involving melanoma recurrence is conducted and reported. Our model is implemented in our R package aftQnp which is available from https://github.com/Isabellee4555/aftQnP.'], 'CopyrightInformation': '© 2024 Wiley‐VCH GmbH.'}, 'AuthorList': ListElement([DictElement({'Identifier': [], 'AffiliationInfo': [{'Identifier': [], 'Affiliation': 'Brain and Mind Centre, The University of Sydney, Sydney, Australia.'}, {'Identifier': [], 'Affiliation': 'School of Mathematical and Physical Sciences, Macquarie University, Sydney, Australia.'}], 'LastName': 'Li', 'ForeName': 'Isabel', 'Initials': 'I'}, attributes={'ValidYN': 'Y'}), DictElement({'Identifier': [], 'AffiliationInfo': [{'Identifier': [], 'Affiliation': 'School of Mathematical and Physical Sciences, Macquarie University, Sydney, Australia.'}], 'LastName': 'Ma', 'ForeName': 'Jun', 'Initials': 'J'}, attributes={'ValidYN': 'Y'}), DictElement({'Identifier': [StringElement('0000-0002-8136-2294', attributes={'Source': 'ORCID'})], 'AffiliationInfo': [{'Identifier': [], 'Affiliation': 'School of Mathematical and Physical Sciences, Macquarie University, Sydney, Australia.'}, {'Identifier': [], 'Affiliation': \"Laboratoire de Mathématiques et de leurs Applications, Université de Pau et des Pays de l'Adour, Pau, France.\"}], 'LastName': 'Liquet', 'ForeName': 'Benoit', 'Initials': 'B'}, attributes={'ValidYN': 'Y'})], attributes={'CompleteYN': 'Y'}), 'PublicationTypeList': [StringElement('Journal Article', attributes={'UI': 'D016428'})]}, attributes={'PubModel': 'Print'}), 'MedlineJournalInfo': {'Country': 'Germany', 'MedlineTA': 'Biom J', 'NlmUniqueID': '7708048', 'ISSNLinking': '0323-3847'}, 'MeshHeadingList': [{'QualifierName': [], 'DescriptorName': StringElement('Humans', attributes={'UI': 'D006801', 'MajorTopicYN': 'N'})}, {'QualifierName': [StringElement('methods', attributes={'UI': 'Q000379', 'MajorTopicYN': 'N'})], 'DescriptorName': StringElement('Biometry', attributes={'UI': 'D001699', 'MajorTopicYN': 'Y'})}, {'QualifierName': [], 'DescriptorName': StringElement('Survival Analysis', attributes={'UI': 'D016019', 'MajorTopicYN': 'N'})}, {'QualifierName': [], 'DescriptorName': StringElement('Models, Statistical', attributes={'UI': 'D015233', 'MajorTopicYN': 'Y'})}, {'QualifierName': [], 'DescriptorName': StringElement('Time Factors', attributes={'UI': 'D013997', 'MajorTopicYN': 'N'})}, {'QualifierName': [], 'DescriptorName': StringElement('Proportional Hazards Models', attributes={'UI': 'D016016', 'MajorTopicYN': 'N'})}, {'QualifierName': [], 'DescriptorName': StringElement('Likelihood Functions', attributes={'UI': 'D016013', 'MajorTopicYN': 'N'})}, {'QualifierName': [StringElement('drug therapy', attributes={'UI': 'Q000188', 'MajorTopicYN': 'N'})], 'DescriptorName': StringElement('Melanoma', attributes={'UI': 'D008545', 'MajorTopicYN': 'N'})}]}, attributes={'Status': 'MEDLINE', 'Owner': 'NLM', 'IndexingMethod': 'Automated'}), 'PubmedData': {'ReferenceList': [{'ReferenceList': [], 'Reference': [{'Citation': 'Buckley, J., and I. James. 1979. “Linear Regression With Censored Data.” Biometrika 66: 429–436.'}, {'Citation': 'Cai, C., Y. Zou, Y. Peng, and J. Zhang. 2012. “smcure: An R‐Package for Estimating Semiparametric Mixture Cure Models.” Computer Methods and Programs in Biomedicine 108: 1255–1260.'}, {'Citation': 'Chan, R. H., and J. Ma. 2012. “A Multiplicative Iterative Algorithm for Box‐Constrained Penalized Likelihood Image Restoration.” IEEE Transactions on Image Processing 21: 3168–3181.'}, {'Citation': 'Green, P. J., and B. W. Silverman. 1993. Nonparametric Regression and Generalized Linear Models: A Roughness Penalty Approach. Chapman and Hall/CRC, New York.'}, {'Citation': \"Grippo, L., F. Lampariello, and S. Lucidi. 1986. “A Nonmonotone Line Search Technique for Newton's Method.” SIAM Journal on Numerical Analysis 23: 707–716.\"}, {'Citation': 'Lai, T. L., and Z. Ying. 1992. “Linear Rank Statistics in Regression Analysis With Censored or Truncated Data.” Journal of Multivariate Analysis 40: 13–45.'}, {'Citation': 'Law, C. G., and R. Brookmeyer. 1992. “Effects of Mid‐Point Imputation on the Analysis of Doubly Censored Data.” Statistics in Medicine 11: 1569–1578.'}, {'Citation': 'Leiter, U., R. Stadler, C. Mauch, et\\xa0al. 2016. “Complete Lymph Node Dissection Versus No Dissection in Patients With Sentinel Lymph Node Biopsy Positive Melanoma (DeCOG‐SLT): A Multicentre, Randomised, Phase 3 Trial.” Lancet Oncology 17: 757–767.'}, {'Citation': 'Leiter, U., R. Stadler, C. Mauch, et\\xa0al. 2019. “Final Analysis of DeCOG‐SLT Trial: No Survival Benefit for Complete Lymph Node Dissection in Patients With Melanoma With Positive Sentinel Node.” Journal of Clinical Oncology 37: 3000–3008.'}, {'Citation': 'Li, J., and J. Ma. 2020. “On Hazard‐Based Penalized Likelihood Estimation of Accelerated Failure Time Model With Partly Interval Censoring.” Statistical Methods in Medical Research 29: 3804–3817.'}, {'Citation': 'Ma, D., J. Ma, and P. Graham. 2023. “On Semiparametric Accelerated Failure Time Models With Time‐Varying Covariates: A Maximum Penalised Likelihood Estimation.” Statistics in Medicine 42, no. 30: 5577–5595. https://doi.org/10.1002/sim.9926.'}, {'Citation': 'Ma, J. 2006. “Multiplicative Algorithms for Maximum Penalized Likelihood Inversion With Non Negative Constraints and Generalized Error Distributions.” Communications in Statistics—Theory and Methods 35: 831–848.'}, {'Citation': 'Ma, J., D.‐L. Couturier, S. Heritier, and I. C. Marschner. 2021. “Penalized Likelihood Estimation of the Proportional Hazards Model for Survival Data With Interval Censoring.” International Journal of Biostatistics 18, no. 2: 553–575.'}, {'Citation': 'Moore, T. J., B. M. Sadler, and R. J. Kozick. 2008. “Maximum‐Likelihood Estimation, the Cramér – Rao Bound, and the Method of Scoring With Parameter Constraints.” IEEE Transactions on Signal Processing 56: 895–908.'}, {'Citation': 'Morton, D. L., J. F. Thompson, and A. J. Cochran. 2014. “Final Trial Report of Sentinel‐Node Biopsy Versus Nodal Observation in Melanoma.” New England Journal of Medicine 370: 599–609.'}, {'Citation': \"Odell, P. M., K. M. Anderson, and R. B. D'Agostino. 1992. “Maximum Likelihood Estimation for Interval‐Censored Data Using a Weibull‐Based Accelerated Failure Time Model.” Biometrics 48, no. 3: 951–959.\"}, {'Citation': 'Peng, Y. 2003. “Estimating Baseline Distribution in Proportional Hazards Cure Models.” Computational Statistics & Data Analysis 42: 187–201.'}, {'Citation': 'Saikia, R., and M. P. Barman. 2017. “A Review on Accelerated Failure Time Models.” International Journal of Statistics and Systems 12: 311–322.'}, {'Citation': 'Sy, J. P., and J. M. Taylor. 2000. “Estimation in a Cox Proportional Hazards Cure Model.” Biometrics 56: 227–236.'}, {'Citation': 'Tsiatis, A. A. 1990. “Estimating Regression Parameters Using Linear Rank Tests for Censored Data.” Annals of Statistics 18, no. 1: 354–372.'}, {'Citation': 'Webb, A., J. Ma, and S. N. Lô. 2022. “Penalized Likelihood Estimation of a Mixture Cure Cox Model With Partly Interval Censoring—An Application to Thin Melanoma.” Statistics in Medicine 41, no. 17: 3260–3280. https://doi.org/10.1002/sim.9415.'}, {'Citation': 'Wei, L.‐J. 1992. “The Accelerated Failure Time Model: A Useful Alternative to the Cox Regression Model in Survival Analysis.” Statistics in Medicine 11: 1871–1879.'}, {'Citation': 'Wong, S. L., M. B. Faries, E. B. Kennedy, et\\xa0al. 2018. “Sentinel Lymph Node Biopsy and Management of Regional Lymph Nodes in Melanoma: American Society of Clinical Oncology and Society of Surgical Oncology Clinical Practice Guideline Update.” Annals of Surgical Oncology 25: 356–377.'}, {'Citation': 'Wood, S. N. 2011. “Fast Stable Restricted Maximum Likelihood and Marginal Likelihood Estimation of Semiparametric Generalized Linear Models.” Journal of the Royal Statistical Society: Series B (Statistical Methodology) 73: 3–36.'}], 'Title': 'References'}], 'History': [DictElement({'Year': '2024', 'Month': '11', 'Day': '7', 'Hour': '6', 'Minute': '24'}, attributes={'PubStatus': 'medline'}), DictElement({'Year': '2024', 'Month': '11', 'Day': '7', 'Hour': '6', 'Minute': '23'}, attributes={'PubStatus': 'pubmed'}), DictElement({'Year': '2024', 'Month': '7', 'Day': '24'}, attributes={'PubStatus': 'revised'}), DictElement({'Year': '2023', 'Month': '7', 'Day': '24'}, attributes={'PubStatus': 'received'}), DictElement({'Year': '2024', 'Month': '8', 'Day': '1'}, attributes={'PubStatus': 'accepted'}), DictElement({'Year': '2024', 'Month': '11', 'Day': '7', 'Hour': '5', 'Minute': '43'}, attributes={'PubStatus': 'entrez'})], 'PublicationStatus': 'ppublish', 'ArticleIdList': [StringElement('39508209', attributes={'IdType': 'pubmed'}), StringElement('10.1002/bimj.202300203', attributes={'IdType': 'doi'})]}}"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# pick a sample to see the output of fetching:\n",
    "article_num = 0\n",
    "records[\"PubmedArticle\"][article_num]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8ddb43b-d99e-4a51-8d00-12d6ac26c537",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "89abc5df-6b37-455b-9010-075c16a99a9e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fedc9d94-5d7d-42a6-b449-d18a36e886ed",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "13265425-ad44-4354-9271-84bce48ead43",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['MedlineCitation', 'PubmedData'])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[\"PubmedArticle\"][article_num].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "188cab89-4200-4a7f-9581-d4793f6c8723",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['OtherID', 'CitationSubset', 'KeywordList', 'OtherAbstract', 'InvestigatorList', 'GeneralNote', 'SpaceFlightMission', 'PMID', 'DateCompleted', 'DateRevised', 'Article', 'MedlineJournalInfo', 'MeshHeadingList'])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[\"PubmedArticle\"][article_num]['MedlineCitation'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9730a520-3e8d-4fb3-bf5d-30f15b1aa607",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['ReferenceList', 'History', 'PublicationStatus', 'ArticleIdList'])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[\"PubmedArticle\"][article_num]['PubmedData'].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "990aa2b5-fa1b-4f01-9a4c-6ba0fbbe48f9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['ELocationID', 'ArticleDate', 'Language', 'Journal', 'ArticleTitle', 'Pagination', 'Abstract', 'AuthorList', 'PublicationTypeList'])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[\"PubmedArticle\"][article_num]['MedlineCitation'][\"Article\"].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "d5cdd235-e246-463c-b317-38f70de02684",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "dict_keys(['AbstractText', 'CopyrightInformation'])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[\"PubmedArticle\"][article_num]['MedlineCitation'][\"Article\"][\"Abstract\"].keys()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "c5fc397c-6698-4ff4-8846-0a7b4bf77e89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mixture Cure Semiparametric Accelerated Failure Time Models With Partly Interval-Censored Data.'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[\"PubmedArticle\"][article_num]['MedlineCitation'][\"Article\"][\"ArticleTitle\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8b8fc60e-17dc-4237-b0c2-a5f697ff46eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['In practical survival analysis, the situation of no event for a patient can arise even after a long period of waiting time, which means a portion of the population may never experience the event of interest. Under this circumstance, one remedy is to adopt a mixture cure Cox model to analyze the survival data. However, if there clearly exhibits an acceleration (or deceleration) factor among their survival times, then an accelerated failure time (AFT) model will be preferred, leading to a mixture cure AFT model. In this paper, we consider a penalized likelihood method to estimate the mixture cure semiparametric AFT models, where the unknown baseline hazard is approximated using Gaussian basis functions. We allow partly interval-censored survival data which can include event times and left-, right-, and interval-censoring times. The penalty function helps to achieve a smooth estimate of the baseline hazard function. We will also provide asymptotic properties to the estimates so that inferences can be made on regression parameters and hazard-related quantities. Simulation studies are conducted to evaluate the model performance, which includes a comparative study with an existing method from the smcure R package. The results show that our proposed penalized likelihood method has acceptable performance in general and produces less bias when faced with the identifiability issue compared to smcure. To illustrate the application of our method, a real case study involving melanoma recurrence is conducted and reported. Our model is implemented in our R package aftQnp which is available from https://github.com/Isabellee4555/aftQnP.']"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "records[\"PubmedArticle\"][article_num]['MedlineCitation'][\"Article\"][\"Abstract\"][\"AbstractText\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1a5ed2e3-1f31-4b00-b4e1-6eb5f0b2a0a7",
   "metadata": {},
   "source": [
    "<a id=\"1\"></a>\n",
    "# <div style=\"text-align:center; border-radius:3px 50px; padding:7px; color:white; margin:0; font-size:100%; font-family:Pacifico; background-color:#a0b1b0; \"><b>⭐ Saving data⭐</b></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "8abd5965-1635-492b-8efa-6853fadb5d5e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7.7% of the data is corrput and failed!\n"
     ]
    }
   ],
   "source": [
    "abstract_bank = []\n",
    "failed_fetech = 0\n",
    "for i in range(num_article):\n",
    "    try:\n",
    "        abstract_text = records[\"PubmedArticle\"][i]['MedlineCitation'][\"Article\"][\"Abstract\"][\"AbstractText\"]\n",
    "        if len(abstract_text)>1: # get rid of those abstracts which has more than 1 part. They are not of standrd order.\n",
    "            continue\n",
    "    except:\n",
    "        # print('one article is not fetched!')\n",
    "        failed_fetech+=1\n",
    "    abstract_bank.append(abstract_text[0])\n",
    "    # print(abstract_text[0])\n",
    "print(f\"{failed_fetech/num_article *100}% of the data is corrput and failed!\")    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "303b99fb-e8fc-4d40-9aed-99cc51ac68d2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "e721df5f-4d22-41b3-88f4-943304c29dd0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fetched articles saved to 'pubmed_skin_cancer_articles.txt'\n"
     ]
    }
   ],
   "source": [
    "with open(\"pubmed_skin_cancer_articles.txt\", \"w\") as f:\n",
    "    for abstract in abstract_bank:\n",
    "        f.write(abstract + \"\\n\\n\")\n",
    "print(\"Fetched articles saved to 'pubmed_skin_cancer_articles.txt'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "c834cdb2-240c-4959-8ae1-7fa2de52999d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "997"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(records[\"PubmedArticle\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "a1b690c8-f821-404a-970e-699fe66a6982",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install transformers datasets\n",
    "# !pip install torch torchvision torchaudio\n",
    "# !pip install accelerate\n",
    "# !pip install --upgrade accelerate\n",
    "# !pip install --upgrade transformers\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a74dea68-c20a-482e-a407-c8d1b93227f2",
   "metadata": {},
   "source": [
    "<a id=\"1\"></a>\n",
    "# <div style=\"text-align:center; border-radius:3px 50px; padding:7px; color:white; margin:0; font-size:100%; font-family:Pacifico; background-color:#a0b1b0; \"><b>⭐ Fine-tune the GPT-2 ⭐</b></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "5a489c90-0460-497b-b26d-4df28caefa9c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import accelerate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "id": "2fa4c5c1-0660-4743-b2af-5d5b7f128148",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='1200' max='1200' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [1200/1200 03:22, Epoch 300/300]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Step</th>\n",
       "      <th>Training Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>500</td>\n",
       "      <td>0.274700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1000</td>\n",
       "      <td>0.014900</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fine-tuning complete. Model saved to './fine_tuned_gpt2'\n"
     ]
    }
   ],
   "source": [
    "from transformers import GPT2Tokenizer, GPT2LMHeadModel, TextDataset, DataCollatorForLanguageModeling, Trainer, TrainingArguments\n",
    "\n",
    "# Load GPT-2 tokenizer and model\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(\"gpt2\")\n",
    "model = GPT2LMHeadModel.from_pretrained(\"gpt2\")\n",
    "\n",
    "# Tokenize the dataset\n",
    "def load_dataset(file_path, block_size=128):\n",
    "    \"\"\"\n",
    "    Load the dataset and tokenize it for GPT-2\n",
    "    \"\"\"\n",
    "    dataset = TextDataset(\n",
    "        tokenizer=tokenizer,\n",
    "        file_path=file_path,\n",
    "        block_size=block_size,\n",
    "    )\n",
    "    return dataset\n",
    "\n",
    "# Data collator for dynamic padding\n",
    "data_collator = DataCollatorForLanguageModeling(\n",
    "    tokenizer=tokenizer,\n",
    "    mlm=False,  # GPT-2 uses causal language modeling, not masked language modeling (MLM)\n",
    ")\n",
    "\n",
    "# Load the custom dataset (skin cancer-related PubMed abstracts)\n",
    "dataset = load_dataset(\"pubmed_skin_cancer_articles.txt\")\n",
    "\n",
    "# Fine-tuning setup using Trainer API\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./fine_tuned_gpt2\",\n",
    "    overwrite_output_dir=True,\n",
    "    num_train_epochs=300,  # Adjust epochs depending on dataset size\n",
    "    per_device_train_batch_size=2,\n",
    "    save_steps=10_000,\n",
    "    save_total_limit=2,\n",
    "    prediction_loss_only=True,\n",
    ")\n",
    "\n",
    "# Trainer API for fine-tuning\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    data_collator=data_collator,\n",
    "    train_dataset=dataset,\n",
    ")\n",
    "\n",
    "# Start fine-tuning\n",
    "trainer.train()\n",
    "\n",
    "# Save the fine-tuned model\n",
    "model.save_pretrained(\"./fine_tuned_gpt2\")\n",
    "tokenizer.save_pretrained(\"./fine_tuned_gpt2\")\n",
    "\n",
    "print(\"Fine-tuning complete. Model saved to './fine_tuned_gpt2'\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cd7b7a20-65de-45fb-8d79-53c8d26a7512",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bb2e6738-b9e7-42a6-ae24-59c13705ef81",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5557e226-6ea7-49a3-8258-daf71c087282",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "945f3ac7-0af8-46c3-b826-eb43ed5421dc",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "62e0c066-1795-4ca7-b095-a6340c563058",
   "metadata": {},
   "source": [
    "<a id=\"1\"></a>\n",
    "# <div style=\"text-align:center; border-radius:3px 50px; padding:7px; color:white; margin:0; font-size:100%; font-family:Pacifico; background-color:#a0b1b0; \"><b>⭐ Inference and Test⭐</b></div>\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "id": "2ca1084c-afb2-46e9-930a-4c5b7673fdbb",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generated Response: What's the skin cancer?\n",
      "\n",
      "Pheochromocytoma, also called macrophage hyperplasia, is a rare cutaneous neuroendocrine cancer that has the potential to metastasize. However, brain metastasis is infrequent in this type of cancer. To illustrate the importance of molecular diagnostic approaches in detecting this rare condition, a real case study involving melanoma recurrence is conducted and reported. Our model is implemented in our R package aftQnp which is available from https://github.com/Isabellee4555/aftQnP.\n",
      " Merkel cell carcinoma (MMC) is one of the important causes of death in patients with MEN2A. MCC is aggressive and has a high median survival. Although no significant difference was observed between treated and untreated patients, an increased likelihood of metastatic melanomas was noted in treated MEN1A patients. The results highlight the need for further investigation and understanding of MEN4A symptoms and indicate\n"
     ]
    }
   ],
   "source": [
    "\n",
    "\n",
    "import torch\n",
    "from transformers import GPT2LMHeadModel, GPT2Tokenizer\n",
    "\n",
    "# Load the fine-tuned model and tokenizer\n",
    "tokenizer = GPT2Tokenizer.from_pretrained(\"./fine_tuned_gpt2\")\n",
    "model = GPT2LMHeadModel.from_pretrained(\"./fine_tuned_gpt2\")\n",
    "\n",
    "# Function to generate a response from the fine-tuned model\n",
    "def generate_finetuned_response(prompt, max_length=200, temperature=0.7):\n",
    "    inputs = tokenizer(prompt, return_tensors='pt', max_length=512, truncation=True)\n",
    "\n",
    "    with torch.no_grad():\n",
    "        outputs = model.generate(\n",
    "            inputs['input_ids'],\n",
    "            max_length=max_length,\n",
    "            temperature=temperature,\n",
    "            num_return_sequences=1,\n",
    "            no_repeat_ngram_size=2,\n",
    "            pad_token_id=tokenizer.eos_token_id\n",
    "        )\n",
    "\n",
    "    return tokenizer.decode(outputs[0], skip_special_tokens=True)\n",
    "\n",
    "# Example: Generate a response to a medical query\n",
    "prompt = \"What's the skin cancer?\"\n",
    "response = generate_finetuned_response(prompt)\n",
    "print(\"Generated Response:\", response)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afad4a75-bf06-4805-90ee-05d6b2819768",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}