주식-뉴스 머신러닝-라벨변경2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "77d4d1da",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "\n",
    "df= pd.read_csv(\"./전처리/df_stock2.csv\")\n",
    "df_stock = df.drop(['Unnamed: 0','Code'],axis = 1)\n",
    "\n",
    "# 1월3일의 결과를 1월 2일의 기사를 토대로 예측해야 하므로 한칸 위로\n",
    "df_stock['target']=df_stock['label'].shift(-1)\n",
    "# shift가 발생하면서 1월2일의 라벨들이 12월 30이롤 이동하기 때문에 삭제\n",
    "check = (df_stock['Date'] == '2019-12-30')\n",
    "# ~ 기호를 이용하여, 해당하지 않은 데이터만 남긴다\n",
    "df_stock = df_stock[~check]\n",
    "\n",
    "# 라벨인코딩을 통해서, 해당 타겟 값을 바꿔준다.\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(df_stock['target'])\n",
    "labels = encoder.transform(df_stock['target'])\n",
    "df_stock['target'] = labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "bd04b333",
   "metadata": {},
   "outputs": [],
   "source": [
    "# head - feature 500\n",
    "df_news = pd.read_csv('./전처리/TF-IDF/head_tf-idf_500.csv')\n",
    "df_stock['Date'] = df_stock['Date'].str.replace('-','.')\n",
    "df_stock['id'] = df_stock['Name']+df_stock['Date']\n",
    "df_news['id'] = df_news['Company']+df_news['Day']\n",
    "total_df = pd.merge(df_stock, df_news, on = 'id').drop(['Name','Date','id','Company','Day','Unnamed: 0','label'],axis=1)\n",
    "total_df = total_df.dropna()\n",
    "target = total_df['target']\n",
    "test = total_df.drop(['target'], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "bce20a55",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(test, target, test_size=0.3, shuffle=False, random_state=100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "f96aa7b3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7689176967138981\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.metrics import accuracy_score\n",
    "\n",
    "lr_clf = LogisticRegression()\n",
    "lr_clf.fit(X_train, y_train)\n",
    "pred = lr_clf.predict(X_test)\n",
    "print(accuracy_score(y_test, pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "90e08437",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <style>\n",
       "    table.eli5-weights tr:hover {\n",
       "        filter: brightness(85%);\n",
       "    }\n",
       "</style>\n",
       "\n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "        <table class=\"eli5-weights eli5-feature-importances\" style=\"border-collapse: collapse; border: none; margin-top: 0em; table-layout: auto;\">\n",
       "    <thead>\n",
       "    <tr style=\"border: none;\">\n",
       "        <th style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">Weight</th>\n",
       "        <th style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">Feature</th>\n",
       "    </tr>\n",
       "    </thead>\n",
       "    <tbody>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 80.00%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0021\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                총수\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 85.60%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0013\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                거래\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 85.61%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0013\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                논란\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 85.61%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0013\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                전일\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 85.61%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0013\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                특별\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 85.61%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0013\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                연봉\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 85.61%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0013\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                업계\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 85.62%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0013\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                최태원\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 87.68%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0010\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                기대\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 87.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0010\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                개선\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 87.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0010\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                사업\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 87.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0010\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                비중\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 87.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0010\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                화학\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 87.69%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0010\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                시장\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 87.70%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0010\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                순매도\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 87.70%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0010\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                선임\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 89.93%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0008\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                하루\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 89.94%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0008\n",
       "                \n",
       "                    &plusmn; 0.0021\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                주총\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.40%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                구조\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.41%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0021\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                확대\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.42%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                쇼핑\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.42%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                참석\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.42%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                카드\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.43%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                증시\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.43%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                수준\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.43%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0013\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                인기\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 92.43%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0005\n",
       "                \n",
       "                    &plusmn; 0.0021\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                마감\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.29%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0019\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                이노베이션\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.30%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                부진\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.32%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                사회\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.32%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                분쟁\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.32%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                그룹\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                오픈\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                기록\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                효과\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                분양\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                강남\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                전략\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                오늘\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                선물\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                간담회\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                최저\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                전문\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                이익\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                스마트\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                예약\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                상승세\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                상장\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                지분\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                통신\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                신규\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                보험\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                벤처\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 95.34%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0003\n",
       "                \n",
       "                    &plusmn; 0.0010\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                금융\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.78%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                하락\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.83%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                모델\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.84%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                조선\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.86%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                대형\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.86%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                가구\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.86%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                화재\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.86%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                조성\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.86%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                악재\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.86%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                오전\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.88%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                식품\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.88%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                정책\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.88%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                가능\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.88%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                팔자\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.88%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                여성\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.88%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                트리\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.88%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                디지털\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.90%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                연말\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.90%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                최대\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.90%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                중공업\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.90%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                실적\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.93%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                특허\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.93%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                준비\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.93%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                거래일\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.93%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                글로벌\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.93%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                합병\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "        <tr style=\"background-color: hsl(120, 100.00%, 99.93%); border: none;\">\n",
       "            <td style=\"padding: 0 1em 0 0.5em; text-align: right; border: none;\">\n",
       "                0.0000\n",
       "                \n",
       "                    &plusmn; 0.0000\n",
       "                \n",
       "            </td>\n",
       "            <td style=\"padding: 0 0.5em 0 0.5em; text-align: left; border: none;\">\n",
       "                신고\n",
       "            </td>\n",
       "        </tr>\n",
       "    \n",
       "    \n",
       "        \n",
       "            <tr style=\"background-color: hsl(120, 100.00%, 99.93%); border: none;\">\n",
       "                <td colspan=\"2\" style=\"padding: 0 0.5em 0 0.5em; text-align: center; border: none; white-space: nowrap;\">\n",
       "                    <i>&hellip; 420 more &hellip;</i>\n",
       "                </td>\n",
       "            </tr>\n",
       "        \n",
       "    \n",
       "    </tbody>\n",
       "</table>\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "    \n",
       "\n",
       "\n",
       "\n"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import eli5\n",
    "from eli5.sklearn  import PermutationImportance\n",
    "\n",
    "perm1 = PermutationImportance(lr_clf, scoring = \"f1\", random_state = 42).fit(X_test, y_test)\n",
    "eli5.show_weights(perm1, top = 80, feature_names = X_test.columns.tolist())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "4a92c0ab",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "오차 행렬\n",
      " [[5099    5]\n",
      " [1528    2]]\n",
      "정확도 :  0.7689176967138981\n",
      "정밀도 :  0.2857142857142857\n",
      "재현율 :  0.00130718954248366\n",
      "f1 :  0.002602472348731295\n",
      "roc :  0.5001637828590161\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics import confusion_matrix, precision_score, recall_score, f1_score, roc_auc_score\n",
    "\n",
    "def get_clf_eval(y_test, pred):\n",
    "    print(\"오차 행렬\\n\", confusion_matrix(y_test, pred))\n",
    "    print(\"정확도 : \", accuracy_score(y_test, pred))\n",
    "    print(\"정밀도 : \", precision_score(y_test, pred))\n",
    "    print(\"재현율 : \", recall_score(y_test, pred))\n",
    "    print(\"f1 : \", f1_score(y_test,pred))\n",
    "    print(\"roc : \", roc_auc_score(y_test,pred))\n",
    "get_clf_eval(y_test, pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "66f6be09",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.823208</td>\n",
       "      <td>0.176792</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.794033</td>\n",
       "      <td>0.205967</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.811164</td>\n",
       "      <td>0.188836</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.799313</td>\n",
       "      <td>0.200687</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.892953</td>\n",
       "      <td>0.107047</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6629</th>\n",
       "      <td>0.832936</td>\n",
       "      <td>0.167064</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6630</th>\n",
       "      <td>0.760999</td>\n",
       "      <td>0.239001</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6631</th>\n",
       "      <td>0.733540</td>\n",
       "      <td>0.266460</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6632</th>\n",
       "      <td>0.721514</td>\n",
       "      <td>0.278486</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6633</th>\n",
       "      <td>0.709012</td>\n",
       "      <td>0.290988</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>6634 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "             0         1    2\n",
       "0     0.823208  0.176792  0.0\n",
       "1     0.794033  0.205967  0.0\n",
       "2     0.811164  0.188836  0.0\n",
       "3     0.799313  0.200687  0.0\n",
       "4     0.892953  0.107047  0.0\n",
       "...        ...       ...  ...\n",
       "6629  0.832936  0.167064  0.0\n",
       "6630  0.760999  0.239001  0.0\n",
       "6631  0.733540  0.266460  0.0\n",
       "6632  0.721514  0.278486  0.0\n",
       "6633  0.709012  0.290988  0.0\n",
       "\n",
       "[6634 rows x 3 columns]"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "pred_proba = lr_clf.predict_proba(X_test) # 각 결과의 대한 예측 확률\n",
    "pred = lr_clf.predict(X_test)\n",
    "pred_proba_result = np.concatenate([pred_proba, pred.reshape(-1,1)], axis = 1) \n",
    "df_proba = pd.DataFrame(pred_proba_result)\n",
    "df_proba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "id": "dafd2d77",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAAM30lEQVR4nO3dX4xcdRnG8edxS2OLtO7aVaCtthf4pzGicYQYoyAEKJKIJl7QEo3GhDQRozcGvFBAbjRB4wWFpsGGmBiaGIqiQWli/BOjmE4V1NJgNiWUtQiLboJSaq19vZhZOp3O7p6pZ+bseef7STY7c+Z0520o3z37m3N2HBECANTfa6oeAABQDoIOAEkQdABIgqADQBIEHQCSWFbVE69ZsyY2bNhQ1dMDQC3t37//xYiY7PVYZUHfsGGDms1mVU8PALVk+5n5HmPJBQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEpVdWISzY7vvP8PvvAdGA0GvmfnibJtwAyOOJRcASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0AShYJue7Ptp2xP2b61x+Orbf/I9hO2D9j+TPmjAgAWsmjQbY9J2i7pWkmbJG2xvalrt89JejIiLpZ0uaRv2l5e8qwjY2JiQrb7+pDU1/4TExMV/y0BlG1ZgX0ukTQVEYckyfZuSddLerJjn5B0nltleZ2kf0g6UfKsI2N2dlYRMdDnmPsmACCPIksuayU923F/ur2t092S3iHpiKQ/SfpCRJzs/kK2b7LdtN2cmZk5y5EBAL0UCXqvQ7nuw8drJD0u6UJJ75Z0t+1VZ/yhiJ0R0YiIxuTkZJ+jAgAWUiTo05LWd9xfp9aReKfPSNoTLVOSnpb09nJGBAAUUSTo+yRdZHtj+4XOGyQ93LXPYUlXSpLtN0l6m6RDZQ4KAFjYoi+KRsQJ2zdLelTSmKRdEXHA9rb24zsk3Snpftt/UmuJ5paIeHGAcwMAuhQ5y0UR8YikR7q27ei4fUTS1eWOBgDoB1eKAkASBB0Akii05ILhittWSbevHvxzAEiFoC9BvuOloVwpGrcP9CkADBlLLgCQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEV4ouUYN+z8/x8fGBfn0Aw0fQl6Czuezf9sB/XQCApY0lFwBIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAoF3fZm20/ZnrJ96zz7XG77cdsHbP+y3DEBAItZttgOtsckbZd0laRpSftsPxwRT3bs83pJ90jaHBGHbb9xQPMCAOZR5Aj9EklTEXEoIo5L2i3p+q59tkraExGHJSkiXih3TADAYooEfa2kZzvuT7e3dXqrpHHbv7C93/anen0h2zfZbtpuzszMnN3EAICeigTdPbZF1/1lkt4r6TpJ10j6iu23nvGHInZGRCMiGpOTk30PCwCYX5GgT0ta33F/naQjPfb5aUS8HBEvSvqVpIvLGRGL+frXJemk7rqr6kkAVKlI0PdJusj2RtvLJd0g6eGufX4o6YO2l9leKelSSQfLHRXz+fKXW5+/9KVq5wBQrUXPcomIE7ZvlvSopDFJuyLigO1t7cd3RMRB2z+V9EdJJyXdFxF/HuTgkHzaYphP2xbdi2IA0nNU9H9+o9GIZrNZyXPXmU+r+EnN/xLHqR++qvpvDKB8tvdHRKPXY4seoWNp6Yyze7W89QgRB0YQl/4DQBIEHQCSIOgAkARBBzAwzz0nXXaZ9Le/VT3JaCDoAAbmzjulX/9a+trXqp5kNBB0AKVbsaJ1Fta990onT7Y+263tGByCDqB0hw5JW7eeCviKFdKNN0pPP13tXNkR9Br7wx96b3/iieHOAXS74AJp1SrplVda9195pXX//POrnSs7rhStsfkvLOLSf1RrxQrp2LEzt7/2tacij7Oz0JWiHKEDKN3cksvKla37K1ey5DIMBL3Gzjmnv+3AsMwtuRw92rp/9ChLLsNA0GvsP//pbzswTM8/Ly1f3rq9fDnnog8Dv5wLQOm6X985flx66KHWdl7fGRyO0Gts7uin6HZgWPbuPbV+Pufcc6Wf/ayaeUYFQa+x+c5yWejsF2AYPvrRU+vnc15+WbruumrmGRUEvcb+/e/+tgPDMt+yCsstg0XQa4yzXLBUcbBRDYJeY888c+byii0dPlzNPMCcvXt7b2cNfbAIeo1deOGZP8JGtM4BBqp09dW9t1955XDnGDUEvcb27pXGxk7fNjbGURAwqgh6jV11Vevqu06rVklXXFHNPMAcXt+pBhcW1dzRo9Lq1dL4uDQ7e+apYkAVuIq5Ghyh19yxY61fgnT4cOtzr99wB2A0EPQa411hsFTxu/qrQdBrjHeFwVL1/vf33n7ppcOdY9QQ9BrjXWGwVM239MeS4GAR9BpbsULaseP0bffey5ILMKoIeo3xrjAAOhH0Gptbcjl2rPVejceOseQCjDKCXnPPPy9t2yY99ljrM+8KA4wuLiyquT17Tt3evr26OQBUjyN0AEiCoANAEgQdAJIg6ABKYfvVD+n7kqL9oY7bD5y2n3kD3FLxoiiAUkTXu62canVIcnufLZK2DHOskVLoCN32ZttP2Z6yfesC+73P9n9tf6K8EQHU27NVDzAyFg267TFJ2yVdK2mTpC22N82z3zckPVr2kADqJ2LuLRLf0nEbg1TkCP0SSVMRcSgijkvaLen6Hvt9XtKDkl4ocT4AQEFFgr5Wp//MNN3e9irbayV9XFLXr4o6ne2bbDdtN2dmZvqdFQCwgCJB7/UydPcPT9+WdEtE/HehLxQROyOiERGNycnJgiMCAIoocpbLtKT1HffXSTrStU9D0u72KUhrJH3E9omI+EEZQwIAFlck6PskXWR7o6S/SrpB0tbOHSJi49xt2/dL+jExB4DhWjToEXHC9s1qnb0yJmlXRBywva39+ILr5gCA4Sh0YVFEPCLpka5tPUMeEZ/+/8cCAPSLS/8BIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdQF8mJiZku/CHpL72t62JiYmK/5b1tKzqAQDUy+zsrCJioM8x940A/eEIHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJLgSlEAfYnbVkm3rx78c6BvBB1AX3zHS0O59D9uH+hTpMSSCwAkQdABIAmCDgBJFAq67c22n7I9ZfvWHo/faPuP7Y/f2L64/FEBAAtZNOi2xyRtl3StpE2Sttje1LXb05Iui4h3SbpT0s6yBwUALKzIEfolkqYi4lBEHJe0W9L1nTtExG8iYrZ99zFJ68odEwCwmCJBXyvp2Y770+1t8/mspJ/0esD2TbabtpszMzPFpwQALKpI0Hu9F1TPk1Btf1itoN/S6/GI2BkRjYhoTE5OFp8SALCoIhcWTUta33F/naQj3TvZfpek+yRdGxF/L2c8AEvRoN/zc3x8fKBfP6siQd8n6SLbGyX9VdINkrZ27mD7zZL2SPpkRPyl9CkBLBn9XiVqe+BXlqJl0aBHxAnbN0t6VNKYpF0RccD2tvbjOyR9VdIbJN3T/s59IiIagxsbANDNVX3nbDQa0Ww2K3luAMPDEXq5bO+f74CZK0UBIAmCDgBJEHQASIKgA0ASBB0AkiDoAJAEQQeAJAg6ACRB0AEgCYIOAEkQdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASAJgg4ASRB0AEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIIllVQ8AIAfbZ/VYRAxinJFE0AGUgjBXjyUXAEiCoANAEgQdAJIg6ACQBEEHgCQIOgAkQdABIAmCDgBJuKqLAWzPSHqmkifPaY2kF6seAuiBf5vlektETPZ6oLKgo1y2mxHRqHoOoBv/NoeHJRcASIKgA0ASBD2PnVUPAMyDf5tDwho6ACTBEToAJEHQASAJgl5ztnfZfsH2n6ueBehke73tn9s+aPuA7S9UPVN2rKHXnO0PSfqXpO9GxDurngeYY/sCSRdExO9tnydpv6SPRcSTFY+WFkfoNRcRv5L0j6rnALpFxHMR8fv27X9KOihpbbVT5UbQAQyc7Q2S3iPpdxWPkhpBBzBQtl8n6UFJX4yIl6qeJzOCDmBgbJ+jVsy/FxF7qp4nO4IOYCBsW9J3JB2MiG9VPc8oIOg1Z/sBSb+V9Dbb07Y/W/VMQNsHJH1S0hW2H29/fKTqoTLjtEUASIIjdABIgqADQBIEHQCSIOgAkARBB4AkCDoAJEHQASCJ/wE6X/ZVhxNX9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "\n",
    "plt.rcParams['figure.figsize'] = [10, 6]\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.boxplot([df_proba[0], df_proba[1]], sym=\"b*\")\n",
    "\n",
    "plt.xticks([1, 2])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "id": "6065e516",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<function matplotlib.pyplot.show(close=None, block=None)>"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAAANdUlEQVR4nO3df4zkd13H8eeLay9SqBTtCoZynDVYQRJaXEtII9ZW9FDBaPpHa4TQlGw0YjAxKvIHxvgPxsTQRKPZFKxELCq0ig1UGkitGCjda68/rzWlIJygt5VoaTU2bd/+MXP1ut29+e7dfGc+M9/nI9l0Zufbyftzs/vaz7y/n893UlVIktr1vHkXIEk6MYNakhpnUEtS4wxqSWqcQS1JjTutjyc9++yza//+/X08tSQtpYMHDz5SVSvbPTYxqJOcB/zlcd86F3hfVX1gp/9n//79bGxs7LZOSRqsJP+y02MTg7qqHgTOHz/RHuBfgRumVZwk6cR226O+FPhSVe2Y/JKk6dptUF8OXNdHIZKk7XUO6iR7gbcCf73D42tJNpJsbG5uTqs+SRq83cyo3wzcUVX/vt2DVbVeVatVtbqysu2JS0nSSdhNUF+BbQ9JmrlOQZ3kDOBNwPX9liNJ2qrThpeq+m/gO3uuRZK0DbeQS1LjetlCLunkXHXt7c/c/uA7fmiOlaglzqglqXEGtSQ1zqCWpMYZ1JLUOINakhpnUEtS4wxqSWqcQS1JjTOoJalx7kyUGuUuRR3jjFqSGmdQS1LjDGpJapxBLUmNM6glqXGu+pAWmCtDhsEZtSQ1zqCWpMYZ1JLUOHvU0hwd32OWdtJpRp3krCQfS/JAksNJ3tB3YZKkka4z6quBm6rqsiR7gTN6rEmSdJyJQZ3k24E3Au8AqKongCf6LUuSdEyX1se5wCbwp0nuTHJNkhdsPSjJWpKNJBubm5tTL1SShqpLUJ8GvA7446q6AHgceM/Wg6pqvapWq2p1ZWVlymVK0nB1CeojwJGqum18/2OMgluSNAMTe9RV9W9JvpbkvKp6ELgUuL//0iQds9ut4m4tXy5dV338CvCR8YqPh4Er+ytJknS8TkFdVYeA1X5LkXQq3DyzvNyZKM2ArQidCq/1IUmNM6glqXG2PqQFYy96eJxRS1LjnFFLA+JJzcXkjFqSGmdQS1LjDGpJapw9ammK7AGrD86oJalxBrUkNc6glqTGGdSS1DhPJkoD5YnPxeGMWpIaZ1BLUuMMaklqnD1qqSetXI60lTp08pxRS1LjDGpJalyn1keSrwDfAp4CnqwqP5FckmZkNz3qH62qR3qrRJK0LVsfktS4rkFdwKeTHEyytt0BSdaSbCTZ2NzcnF6FkjRwXYP6oqp6HfBm4JeTvHHrAVW1XlWrVbW6srIy1SIlacg69air6uvj/x5NcgNwIXBrn4VJy8p1zdqtiTPqJC9Icuax28CPA/f2XZgkaaTLjPolwA1Jjh3/F1V1U69VSZKeMTGoq+ph4LUzqEWStA2X50lS4wxqSWqcV8+T5Ke9NM6glvQshnZ7bH1IUuMMaklqnEEtSY0zqCWpcZ5MlDra6STbUK7d4UnG+XFGLUmNM6glqXEGtSQ1zqCWpMYZ1JLUOINakhrn8jzpJAxlSZ7a4IxakhpnUEtS4wxqSWqcPWppC7dKqzUGtXQCnjTc3k7/Lv5h64etD0lqXOegTrInyZ1JbuyzIEnSs+1mRv1u4HBfhUiSttcpqJOcA/wUcE2/5UiStup6MvEDwG8AZ+50QJI1YA1g3759p1yYpPnzZGobJs6ok/w0cLSqDp7ouKpar6rVqlpdWVmZWoGSNHRdWh8XAW9N8hXgo8AlSf6816okSc+YGNRV9VtVdU5V7QcuBz5bVb/Qe2WSJMB11JLUvF3tTKyqW4BbeqlEkrQtZ9SS1DiDWpIaZ1BLUuMMaklqnEEtSY0zqCWpcX5wgKRe+Ek50+OMWpIaZ1BLUuNsfUh4OU+1zaDWYBnOWhS2PiSpcQa1JDXOoJakxhnUktQ4TyZKmhpP0PbDGbUkNc6glqTG2fqQ1Duv+3FqnFFLUuMMaklq3MSgTvJtSb6Y5K4k9yX5nVkUJkka6dKj/l/gkqp6LMnpwOeSfKqqvtBzbdLUuXxs/uxX797EoK6qAh4b3z19/FV9FiVNk+GsRdepR51kT5JDwFHg5qq6bZtj1pJsJNnY3NyccpmSNFydgrqqnqqq84FzgAuTvGabY9ararWqVldWVqZcpiQN165WfVTVfwK3AAf6KEaS9FxdVn2sJDlrfPv5wI8BD/RclyRprMuqj+8G/izJHkbB/ldVdWO/ZUmSjumy6uNu4IIZ1CJJ2oY7EyWpcQa1JDXOq+dpobnLTUNgUGtpGNpaVrY+JKlxzqi1cLx2h4bGGbUkNc6glqTGGdSS1Dh71FpK9rG1TJxRS1LjDGpJapxBLUmNM6glqXEGtSQ1zqCWpMYZ1JLUOINakhpnUEtS4wxqSWqcQS1JjTOoJalxEy/KlOTlwIeBlwJPA+tVdXXfhUlafn58Wjddrp73JPBrVXVHkjOBg0lurqr7e65NkkSHoK6qbwDfGN/+VpLDwMsAg1pSb5xt/79d9aiT7AcuAG7b5rG1JBtJNjY3N6dUniSpc1AneSHwceBXq+rRrY9X1XpVrVbV6srKyjRrlKRB6/QJL0lOZxTSH6mq6/stSRrxra80MnFGnSTAB4HDVfUH/ZckSTpelxn1RcDbgHuSHBp/771V9cneqpK28DMQl5+v8c66rPr4HJAZ1KKBssUhnZg7EyWpcQa1JDXOoJakxhnUktS4TuuoJWmehn7C2Rm1JDXOGbWa4lpa6bmcUUtS4wxqSWqcQS1JjTOoJalxnkyUtLCGsmzPGbUkNc6glqTGGdSS1DiDWpIaZ1BLUuNc9SFpoQzxMgMGteZiiL9s0skyqNUbw1iaDnvUktS4iUGd5ENJjia5dxYFSZKercuM+lrgQM91SJJ2MLFHXVW3Jtk/g1q0BOxLS9M3tZOJSdaANYB9+/ZN62klqZNlvkDT1IK6qtaBdYDV1dWa1vOqTcv8SyG1xlUfktQ411FLWjrL9o6vy/K864DPA+clOZLkqv7LkiQd02XVxxWzKETt2GnlxjLMTKRFZI9akhpnj1qduUZai2gZ+tUGtU6ZAS71y9aHJDXOoJakxhnUktQ4g1qSGmdQS1LjXPUhwJUbUssM6gEznKXFYFAPjOGsIdtp80vrm2IM6iXS+g+bpJPjyURJapwz6iXl7FpaHga1pEFapPM1BvUC2u1seZF+IKWWtPLO1KBecIawtPwM6oa18tdc0nwZ1JJ0nBbfpRrUM3QqM+QWf3gkzeadr+uoJalxzqin4GT+onbZyipJ0DGokxwArgb2ANdU1ft7rWoKTmUJ2yJdA0DSbMwzCyYGdZI9wB8BbwKOALcn+URV3d93ca3p64VyFi3pRLrMqC8EHqqqhwGSfBT4GaCXoO57ZtslFE/lGENX0rSlqk58QHIZcKCq3jm+/zbg9VX1ri3HrQFr47vnAQ9Ov9zenA08Mu8i5mTIY4dhj9+xt+UVVbWy3QNdZtTZ5nvPSfeqWgfWd1lYE5JsVNXqvOuYhyGPHYY9fse+OGPvsjzvCPDy4+6fA3y9n3IkSVt1CerbgVcm+Z4ke4HLgU/0W5Yk6ZiJrY+qejLJu4C/Z7Q870NVdV/vlc3WQrZspmTIY4dhj9+xL4iJJxMlSfPlFnJJapxBLUmNG1RQJzmQ5MEkDyV5zzaPX5zkv5IcGn+9bx519mHS2MfHXDwe931J/mHWNfalw+v+68e95vcmeSrJd8yj1mnrMPYXJfm7JHeNX/cr51FnXzqM/8VJbkhyd5IvJnnNPOqcqKoG8cXoROiXgHOBvcBdwKu3HHMxcOO8a53T2M9itNt03/j+d8277lmNfcvxbwE+O++6Z/i6vxf4vfHtFeCbwN551z7D8f8+8Nvj298PfGbedW/3NaQZ9TNb4avqCeDYVvgh6DL2nweur6qvAlTV0RnX2Jfdvu5XANfNpLL+dRl7AWcmCfBCRkH95GzL7E2X8b8a+AxAVT0A7E/yktmWOdmQgvplwNeOu39k/L2t3jB+G/ipJD8wm9J612Xs3we8OMktSQ4mefvMqutX19edJGcAB4CPz6CuWegy9j8EXsVoE9s9wLur6unZlNe7LuO/C/g5gCQXAq9gtKmvKUO6HnWXrfB3MNpv/1iSnwT+Bnhl34XNQJexnwb8IHAp8Hzg80m+UFX/3HdxPet0CYSxtwD/VFXf7LGeWeoy9p8ADgGXAN8L3JzkH6vq0Z5rm4Uu438/cHWSQ4z+UN1Jg+8ohjSjnrgVvqoerarHxrc/CZye5OzZldibLpcBOALcVFWPV9UjwK3Aa2dUX592cwmEy1metgd0G/uVjFpeVVUPAV9m1KtdBl1/56+sqvOBtzPq0395ZhV2NKSgnrgVPslLx726Y2+Dngf8x8wrnb4ulwH4W+CHk5w2bgG8Hjg84zr70OkSCEleBPwIo3+HZdFl7F9l9C6KcW/2PODhmVbZny6/82eNHwN4J3Bri+8mBtP6qB22wif5xfHjfwJcBvxSkieB/wEur/Hp4EXWZexVdTjJTcDdwNOMPsnn3vlVPR0dX3eAnwU+XVWPz6nUqes49t8Frk1yD6NWwW+O31EtvI7jfxXw4SRPMVr1dNXcCj4Bt5BLUuOG1PqQpIVkUEtS4wxqSWqcQS1JjTOoJalxBrUkNc6glqTG/R+7Aqx1G/39awAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# 0을 예측하는 확률의 분포\n",
    "plt.hist(df_proba[0], bins=100, density=True, alpha=0.7, histtype='stepfilled')\n",
    "plt.show"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "9e28365f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "임곗값 : 0.2\n",
      "오차 행렬\n",
      " [[1511 3593]\n",
      " [ 423 1107]]\n",
      "정확도 :  0.3946337051552608\n",
      "정밀도 :  0.23553191489361702\n",
      "재현율 :  0.7235294117647059\n",
      "f1 :  0.35537720706260034\n",
      "roc :  0.5097858657569612\n",
      "임곗값 : 0.25\n",
      "오차 행렬\n",
      " [[3036 2068]\n",
      " [ 854  676]]\n",
      "정확도 :  0.5595417545975279\n",
      "정밀도 :  0.24635568513119532\n",
      "재현율 :  0.4418300653594771\n",
      "f1 :  0.31633130556855404\n",
      "roc :  0.5183288257831868\n",
      "임곗값 : 0.3\n",
      "오차 행렬\n",
      " [[4183  921]\n",
      " [1231  299]]\n",
      "정확도 :  0.6756104914078988\n",
      "정밀도 :  0.24508196721311476\n",
      "재현율 :  0.1954248366013072\n",
      "f1 :  0.21745454545454546\n",
      "roc :  0.5074890640686787\n",
      "임곗값 : 0.35\n",
      "오차 행렬\n",
      " [[4778  326]\n",
      " [1425  105]]\n",
      "정확도 :  0.736056677720832\n",
      "정밀도 :  0.24361948955916474\n",
      "재현율 :  0.06862745098039216\n",
      "f1 :  0.10708822029576746\n",
      "roc :  0.5023779888130802\n",
      "임곗값 : 0.4\n",
      "오차 행렬\n",
      " [[5004  100]\n",
      " [1502   28]]\n",
      "정확도 :  0.758516731986735\n",
      "정밀도 :  0.21875\n",
      "재현율 :  0.018300653594771243\n",
      "f1 :  0.03377563329312425\n",
      "roc :  0.4993540885528715\n",
      "임곗값 : 0.45\n",
      "오차 행렬\n",
      " [[5078   26]\n",
      " [1523    7]]\n",
      "정확도 :  0.7665058788061502\n",
      "정밀도 :  0.21212121212121213\n",
      "재현율 :  0.004575163398692811\n",
      "f1 :  0.008957133717210493\n",
      "roc :  0.49974055975577275\n",
      "임곗값 : 0.5\n",
      "오차 행렬\n",
      " [[5099    5]\n",
      " [1528    2]]\n",
      "정확도 :  0.7689176967138981\n",
      "정밀도 :  0.2857142857142857\n",
      "재현율 :  0.00130718954248366\n",
      "f1 :  0.002602472348731295\n",
      "roc :  0.5001637828590161\n",
      "임곗값 : 0.55\n",
      "오차 행렬\n",
      " [[5104    0]\n",
      " [1530    0]]\n",
      "정확도 :  0.7693699125716008\n",
      "정밀도 :  0.0\n",
      "재현율 :  0.0\n",
      "f1 :  0.0\n",
      "roc :  0.5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\gimhu\\anaconda3\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1221: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import Binarizer\n",
    "\n",
    "thresholds = [0.2,0.25,0.3,0.35,0.4,0.45,0.5,0.55]\n",
    "\n",
    "def get_eval_by_threshold(y_test, pred_proba, thresholds):\n",
    "    for custom in thresholds:\n",
    "        binarizer = Binarizer(threshold=custom).fit(pred_proba)\n",
    "        custom_predict = binarizer.transform(pred_proba)\n",
    "        print(f\"임곗값 : {custom}\")\n",
    "        get_clf_eval(y_test, custom_predict)\n",
    "        \n",
    "get_eval_by_threshold(y_test, pred_proba[:,1].reshape(-1,1), thresholds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "id": "4b6ecb74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAFzCAYAAAAuSjCuAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuNCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8QVMy6AAAACXBIWXMAAAsTAAALEwEAmpwYAABTwklEQVR4nO3dd3gVVf7H8fc3PUAILSC9d+kdVIoVddeGYl3bqth/67rq6lp21V3Xsq6uunZR14aKvTdsFCnSuxgg9E5CSD+/P+YmhJCQC2Qyucnn9Tz3mbl35t58cgj5ZmbOnGPOOURERCTyRAUdQERERA6OiriIiEiEUhEXERGJUCriIiIiEUpFXEREJEKpiIuIiESomKADHKh69eq5Dh06BB2j2tm1axe1a9cOOka1o3b1h9rVH2pXf1REu86cOXOzcy6l5OsRV8SbNGnCjBkzgo5R7UyaNIkRI0YEHaPaUbv6Q+3qD7WrPyqiXc1sZWmv63S6iIhIhFIRFxERiVAq4iIiIhEq4q6Ji4hI1ZSbm0taWhpZWVlBR6lSkpOTWbRoUVj7JiQk0KJFC2JjY8PaX0VcREQqRFpaGklJSbRp0wYzCzpOlZGenk5SUlK5+znn2LJlC2lpabRt2zasz9bpdBERqRBZWVk0bNhQBfwgmRkNGzY8oDMZKuIiIlJhVMAPzYG2n4q4iIjIfsyYMYPrrruuzO1r165lzJgxlZhoD10TFxGRGiU/P5/o6Oiw9+/fvz/9+/cvc3uzZs146623KiLaAfPtSNzMnjezjWY2v4ztZmaPmtlyM5trZn39yiIiIjVDamoqXbp04cILL6Rnz56MGTOGzMxM2rRpw9/+9jeOOOII3nzzTT7//HOGDBlC3759OfPMM8nIyABg+vTpDB06lF69ejFw4EDS09OZNGkSJ598MgDffvstvXv3pnfv3vTp04f09HRSU1M5/PDDAa9fwMUXX0yPHj3o06cP33zzDQDjx4/n9NNP54QTTqBjx47cdNNNFfL9+nkkPh54DHipjO2jgY6hxyDgv6GliIhUA2OfmrLPayf3bMoFQ9qwOyefi174aZ/tY/q14Mz+Ldm6K4cr/zdzr21vXDEkrK+7ZMkSnnvuOYYNG8Yll1zCE088AXi3b/3www9s3ryZ008/nS+//JLatWvzz3/+k3/961/ccsstjB07ljfeeIMBAwawc+dOEhMT9/rsBx98kMcff5xhw4aRkZFBQkLCXtsff/xxAObNm8fixYs57rjjmDnT+z5mz57Nzz//THx8PJ07d+baa6+lZcuWYX1PZfHtSNw59x2wdT+7nAK85DxTgXpm1tSvPGH55RtYOxvSN0DGJigoCDSOiIgcuJYtWzJs2DAAzj//fH744QcAxo4dC8DUqVNZuHAhw4YNo3fv3rz44ousXLmSJUuW0LRpUwYMGABA3bp1iYnZ+1h32LBh3HDDDTz66KNs3759n+0//PADF1xwAQBdunShdevWLF++HICjjz6a5ORkEhIS6NatGytXljoc+gEJ8pp4c2B1sedpodfWldzRzC4HLgdISUlh0qRJvgQaMvkS4nP2/N1RYLHkxNUns1ZzcuKSyUpoXJiIjDptyajTjryY2uTF1vElT2XKyMjwrV1rMrWrP9Su/jjUdk1OTiY9Pb3o+bPnHl7qfoX77G97bCnbi392WQpPixfum5mZSX5+Ps45nHOkp6eTmZnJiBEjeOGFF/Z67/z58ykoKNjn62RmZpKXl0d6ejpXX301I0aM4PPPP2fQoEG8//77xMfHF70vNzeXzMzMos/Iz88nPz+frKwszKzodeccO3fuLPV7ysrKCvvfIcgiXlo/elfajs65p4GnATp37ux8m2Wn2yeweQns2gQF+UTtWE3C5uUkpK+FrbMgu7Cxi8c0qNcK6rem6Fsyg7otILkFtBoE7UZ6r1Vhmr3IH2pXf6hd/XGo7bpo0aKwBjXxU506dVi9ejXz589nyJAhvPfee4wYMYJ58+ZRp04dkpKSGDlyJDfeeCMbNmygQ4cOZGZmkpaWRr9+/diwYQOLFy9mwIABpKenk5iYSK1atYiJiSEpKYlffvmFwYMHM3jwYGbNmsXq1avp3bs3UVFRJCUlMWrUKN555x1OPvlkli5dypo1a+jSpQvLly8nLi6uqH1iYmKoVatWqe2VkJBAnz59wvp+gyziaUDxiwEtgLUBZfE07uI9ypOTCaumQPo62LEG1s+FXZv3bE9fCysm7XlerzW0HARH3gCNOkOU7uwTEfFL165defHFF7niiivo2LEjV155Jf/5z3+KtqekpDB+/HjOOeccsrOzAbjnnnvo1KkTb7zxBtdeey27d+8mMTGRL7/8cq/P/ve//80333xDdHQ03bp1Y/To0axbt+cE8lVXXcW4cePo0aMHMTExjB8/nvj4eN++V3Ou1IPfivlwszbAh865fc6ZmNlJwDXAiXgd2h51zg0s7zM7d+7slixZUtFRK152BuxcAyt/hEUfwopvwBVAfF0YeSu0GACJ9aFh+6CTAjqy8Yva1R9qV39UxJF4165dKy7QQUhNTeXkk09m/vxSb4wKRLjDrhYqrR3NbKZzbp/73Hw7Ejez14ARQCMzSwPuBGIBnHNPAh/jFfDlQCZwsV9ZAhFfB1I6e4/+l8CmJbDiW1gwET69Zc9+3U+Drr+FbqfqCF1ERA6Ib0XcOXdOOdsdcLVfX7/KKSzoAy+DtT97p9+n/AcWvOM9ku+ELidCnSaQUBf6X1rlr6OLiFQ1bdq0qVJH4X7TiG2VzQyah8a16XgsZO2A7x/0bm+bOR7yQgPfzxwPox+AlgMhKvyRhUREpOZQEQ+SGSTWg+Pu8Z7n50HmFpj0D5j5ArxwAnQaDWNfhujw5pYVEZGaQxdhq5LoGEhqAr/5N1z6JTTvD0s/gbsbwXtXQ9rMcj9CRERqDhXxqqrlAPjde16HtyY94Of/wbOj4ImhkLs76HQiIlIFqIhXZfF14KwX4cof4Pq50GIgbFwAL50C62tOxw0RkSCNHz+ea665BoC77rqLBx98MOBEe6iIR4r6reH3X8Dxf4fV0+DJYTDx8qBTiYhUWc45Cqr5HBgq4pFmyNUwzhvMn7lvwLtXweTHig0JKyJSc6WmptK1a1euuuoq+vbty913382AAQPo2bMnd955Z9F+L730Ej179qRXr15FE5Z88MEHDBo0iD59+nDMMcewYcOGoL6NsKl3eiQ6rAdc8hl8dhvMexPyc+Cbe+HiT6BZ76DTiYjAJ7fA+nkV+5mH9YDR95W725IlS3jhhRc49dRTeeutt/jpp59wzvHb3/6W7777joYNG3Lvvffy448/0qhRI7Zu9Sa+OuKII5g6dSpmxrPPPsv999/PQw89VLHfQwVTEY9UrQbDZV+Bc7D0U3jzInh6OAz4PZz4oAaKEZEaq3Xr1gwePJgbb7yRzz//vGgykYyMDJYtW8acOXMYM2YMjRo1AqBBgwYApKWlMXbsWNatW0dOTg5t27YN7HsIl4p4pDODzqPhd+/D88fB9GchbTqc8Rw07KBiLiLBCOOI2S+1a9cGvGvif/7zn7niiiv22v7oo49ipfxuvPbaa7nhhhv47W9/y6RJk7jrrrsqI+4h0TXx6qLVILh1rTdO+7o58Fh/uKcxvDMOqnnHDhGR0hx//PE8//zzRXOMr1mzho0bN3L00UczYcIEtmzZAlB0On3Hjh00b94cgBdffDGY0AdIRbw6iasNJz8MF7wLw673rpXPeQ3+Vh9mvQw5u4JOKCJSaY477jjOPfdchgwZQo8ePRgzZgzp6el0796d2267jeHDh9OrVy9uuOEGwLt97Mwzz+TII48sOtVe1fk6FakfImYq0qogPxdePw+WfbbntTZHwtmvepOsFKOpHf2hdvWH2tUf1WEq0qrIz6lIdSRenUXHwnkT4A8Lvc5uMYmQ+j08eQQsfD/odCIicohUxGuC5ObeFKh/WQ9H3QTbV8KEC+CLO7xZ1EREJCKpiNc0o26DPy71Jlf58RG4r5WulYuIRCgV8ZooqQlc+gX0Otd7/vdmtFr5VrCZRKRaiLR+VlXNgbafinhNFRUFpzwG/S4GjHa/vgxvXRJ0KhGJYAkJCWzZskWF/CA559iyZQsJCQlhv0eDvdRkUdHe3OWjbocH2sH8tyEqBk5/OuhkIhKBWrRoQVpaGps2bQo6SpWSlZUVdmFOSEigRYsWYX+2irhA7YZ8e9RbDJ/zB29SleQWcPQdQacSkQgTGxsbEUOVVrZJkyYVDf1a0XQ6XQBwUbHe7GhNDofvH4KtK4KOJCIi5VARlz0S68HYl731R/vAsi8CjSMiIvunIi57a9AOTgpNvffKGFgzK9g8IiJSJhVx2deA38P5E731Z0bCjBeCzSMiIqVSEZfSdTgarp4O0XEw4/mg04iISClUxKVsKZ3guHtg/Vz4RytY/FHQiUREpBgVcdm/XufAgMsgewdMfizoNCIiUoyKuOxfQl046UFvrPVVk2HHmqATiYhIiIq4hOfE+73lk0fAjrRgs4iICKAiLuFq3g8GXwW7t8KrZ0N+XtCJRERqPBVxCd8J/4CTH4YN8+DZo4NOIyJS46mIy4HpexFYFKybDUs+DTqNiEiNpiIuByYqCm5cDhi8NhZWTAo6kYhIjaUiLgeudkM45zVv/aVTIH19sHlERGooFXE5OJ1Hw4kPeuv/6Q/OBZtHRKQGUhGXgzfg92DRkJMO054MOo2ISI2jIi4HzwxuWw/xdeHTW2Dd3KATiYjUKCricmhi4uCc1731p46ErJ3B5hERqUFUxOXQtRkGg6701h/pCaumBptHRKSGUBGXijH6PjjtKdi9DZ4/HratDDqRiEi1pyIuFafX2XDKE976V38NNouISA2gIi4Vq895ULsxLHwf5r0VdBoRkWpNRVwq3tmvQkEuvH8dZG4NOo2ISLWlIi4Vr+UAOO9tyN0FUx4LOo2ISLWlIi7+6HgMtBwE3z8E71wJBQVBJxIRqXZUxMU/ZzzrXR+f8yos+SjoNCIi1Y6KuPinXiu45idvfdZLwWYREamGVMTFX4n1oePxsOxz2LQk6DQiItWKirj4b9AV3vLxgbD8y2CziIhUIyri4r8OR3uDwETFwISLoCA/6EQiItWCr0XczE4wsyVmttzMbille7KZfWBmc8xsgZld7GceCVCf82D4zd60pes125mISEXwrYibWTTwODAa6AacY2bdSux2NbDQOdcLGAE8ZGZxfmWSgHU7xVu+M05H4yIiFcDPI/GBwHLn3ArnXA7wOnBKiX0ckGRmBtQBtgJ5PmaSIKV0htZHwKbF8MxIyNoRdCIRkYjmZxFvDqwu9jwt9FpxjwFdgbXAPOB655xGBanOLvwA2hwJ6+bA6+eBc0EnEhGJWDE+fraV8lrJ39jHA7OBUUB74Asz+945t3OvDzK7HLgcICUlhUmTJlV42JouIyOj8tq1zY303raVeqnfs/bpsSztfFXlfN0AVGq71iBqV3+oXf3hZ7v6WcTTgJbFnrfAO+Iu7mLgPuecA5ab2a9AF+Cn4js5554Gngbo3LmzGzFihF+Za6xJkyZRqe16xDdwT2OarfuMZseMg/ajKu9rV6JKb9caQu3qD7WrP/xsVz9Pp08HOppZ21BntbOB90vsswo4GsDMmgCdgRU+ZpKqIiYervjOW3/795CvrhAiIgfKtyLunMsDrgE+AxYBE5xzC8xsnJmNC+12NzDUzOYBXwE3O+c2+5VJqpimvaDFAMjcAi+fqkIuInKA/DydjnPuY+DjEq89WWx9LXCcnxmkirvoI3i0L6R+D1OfgGHXBZ1IRCRiaMQ2CVZMPFz3s7c+7UnIywk2j4hIBFERl+DFxMGY52HnGnip5FACIiJSFhVxqRoOPwOa9IBVk+GnZ4JOIyISEVTEpeq47GtI6QrfPaBhWUVEwqAiLlVHTBwcdSNkbIAF7wSdRkSkylMRl6ql0wne8u1L4Zdvgs0iIlLFqYhL1RJfB8562Vt/+VTYvT3INCIiVZqKuFQ93X4LR9/hrU9+NNgsIiJVmIq4VE1H/hEatIO5b0J+btBpRESqJBVxqbpG/QV2rIIJFwadRESkSlIRl6qr++mQ2ACWfAQLS86dIyIiKuJSdZnB797z1idcANnpweYREaliVMSlamvaE0580Fv/3xnBZhERqWJUxKXqG3gZtBgIq6fB2tlBpxERqTJUxCUynP60t3z2aHAu2CwiIlWEirhEhgZtoc/5UJAHc14POo2ISJWgIi6R44R/esspjwWbQ0SkilARl8gRXwcad4cN8yH1h6DTiIgETkVcIsvY0Ljq40+CtJnBZhERCZiKuESWhu3hN49465//JdgsIiIBC7uIm1ltP4OIhK3fRdBiAKyaDF/cAXk5QScSEQlEuUXczIaa2UJgUeh5LzN7wvdkIvsz6nawKPjxEVjycdBpREQCEc6R+MPA8cAWAOfcHOAoP0OJlKvdcLhtPSQkw7SndO+4iNRIYZ1Od86tLvFSvg9ZRA5MTDwMv9k7rf7B9UGnERGpdOEU8dVmNhRwZhZnZjcSOrUuEriBV0BSM5j1Iiz6MOg0IiKVKpwiPg64GmgOpAG9Q89FghcdA5d86q3PeD7YLCIilSymvB2cc5uB8yohi8jBqd8aOh4Pyz6DzK1Qq0HQiUREKkW5RdzMXgD26TXknLvEl0QiB6P3OV4Rn/QPOPGBoNOIiFSKcE6nfwh8FHp8BdQFMvwMJXLAup7iLac/B7u2BJtFRKSSlFvEnXNvF3u8ApwFHO5/NJEDEBUFJ/8bXD5MvEy3nIlIjXAww652BFpVdBCRQ9b/Yuh+GvzyFaz9Oeg0IiK+C2fEtnQz21m4BD4AbvY/mshBOPpOb/nMSMjPCzaLiIjPwjmdnuScq1ts2ck593ZlhBM5YA3aQv023vqPDwcaRUTEb2X2Tjezvvt7o3NuVsXHEakA182Gf/eAr++BXudAcougE4mI+GJ/t5g9tJ9tDhhVwVlEKoaZd1p94u/hnXFwkUZyE5Hqqcwi7pwbWZlBRCpUzzNhymOQ+j1M/S8MvjLoRCIiFS6s3ulmdriZnWVmvyt8+B1M5JCd9aK3/PQWyN0dbBYRER+E0zv9TuA/ocdI4H7gtz7nEjl09dvA8Fu89XsPg7ycQOOIiFS0cI7ExwBHA+udcxcDvYB4X1OJVJThN0O30Ghuc14NNouISAULp4jvds4VAHlmVhfYCLTzN5ZIBYmKguPu8dY/uF73jotItRJOEZ9hZvWAZ4CZwCzgJz9DiVSoeq1gxK3e+vPHBZtFRKQChTMV6VWh1SfN7FOgrnNurr+xRCrYUTfC8i8gbTps+QUatg86kYjIIQunY9t7ZnaumdV2zqWqgEtEioqGMc9764t137iIVA/hnE7/F3AEsNDM3jSzMWaW4HMukYpXrxU06wML3ws6iYhIhQhn7PRvQ6fU2wFP401FutHvYCK+6PpbWDMTtq8OOomIyCELd7CXROAMYBwwAHjRz1Aivim83ezRPpCXHWwWEZFDFM418TeARXhjpT8OtHfOXet3MBFfNGwPvc+DglxvJDcRkQgWzpH4C3iFe5xz7uvQPeMikeuUx73ljOchJzPYLCIihyCca+KfOufyKyOMSKUwgxMf9NaXfhpsFhGRQxDWNXGRaqfnWG85+5Vgc4iIHAIVcamZEupC096w/EvI2RV0GhGRg1JmETezvvt7hPPhZnaCmS0xs+VmVmovIjMbYWazzWyBmX17sN+IyAEbdr23/OiPweYQETlI+xt29aH9bHN4vdXLZGbReL3ZjwXSgOlm9r5zbmGxfeoBTwAnOOdWmVnjcIOLHLLDT4f3r4X5b8NvHoEYTc4nIpGlzCLunBt5iJ89EFjunFsBYGavA6cAC4vtcy4w0Tm3KvQ1NYiMVK6jboQv74I5r0G/i4JOIyJyQMos4mZ2+v7e6JybWM5nNweKD4uVBgwqsU8nINbMJgFJwCPOuZdKyXI5cDlASkoKkyZNKudLy4HKyMiome3qejMooQmZP77EvPQ2Ff7xNbZdfaZ29Yfa1R9+tuv+Tqf/Zj/bHFBeEbcy3lfy6/cDjgYSgSlmNtU5t3SvNzn3NN6Qr3Tu3NmNGDGinC8tB2rSpEnU2HbdPIzEBRMZMWwQxCZW6EfX6Hb1kdrVH2pXf/jZrvs7nX7xIX52GtCy2PMWwNpS9tnsnNsF7DKz74BewFJEKkvbo2DBRPjwBjjtv0GnEREJW7hjp59kZjeZ2R2FjzDeNh3oaGZtzSwOOBt4v8Q+7wFHmlmMmdXCO92+6EC+AZFD1vs8bznnVdi9PdAoIiIHIpyx058ExgLX4p0iPxNoXd77nHN5wDXAZ3iFeYJzboGZjTOzcaF9FgGfAnOBn4BnnXPzD/J7ETk4MXFw+jPe+ud/CTaLiMgB2N818UJDnXM9zWyuc+6vZvYQ5V8PB8A59zHwcYnXnizx/AHggXADi/iix5kw8TJv8JeCfIiKDjqRiEi5wjmdvju0zDSzZkAu0Na/SCIBMIPTnoL0dfDN34NOIyISlnCK+IehQVkeAGYBqcDrPmYSCUbPsdDkcPj+QdixJug0IiLlCmcWs7udc9udc2/jXQvv4py73f9oIpXMDI4MDcG68N1Ao4iIhCOcjm1Xh47Ecc5lA1FmdpXfwUQCcfjp0Lgb/Pho0ElERMoVzun0y5xz2wufOOe2AZf5lkgkaC36Q8Z6b0x1EZEqLJwiHmVmRaOvhSY2ifMvkkjA+oXGOXrrEl0bF5EqLZwi/hkwwcyONrNRwGt493aLVE/N+8KlX3rrH/8p2CwiIvsRThG/GfgauBK4GvgKuMnPUCKBazkAep0DSz7y7h0XEamCwumdXgCMB25zzp3hnHvKOZfvezKRoB0ful/8q7uDzSEiUoZweqf/FphN6BS6mfU2s5JjoItUP7UaQJsjYd1s2PJL0GlERPYRzun0O4GBwHYA59xsoI1viUSqkoGXe8uJuiFDRKqecIp4nnNuh+9JRKqibr+F5FawZhbk5QSdRkRkL+EU8flmdi4QbWYdzew/wGSfc4lUHcNvAhxsXBB0EhGRvYRTxK8FugPZeLeX7QCu9zOUSJWS0sVbTnki2BwiIiWE0zs90zl3m3NugHOuP/A/4DH/o4lUES0HQFwdmDdBt5uJSJVSZhE3s55m9rmZzTezu82siZm9DXwJLKy8iCJVwNiXveXr54FzwWYREQnZ35H4M8CrwBnAZrxpSFcAHZxzD1dCNpGqo/0oOOIPkJcFy78KOo2ICLD/Ih7vnBvvnFvinHsEKABucc5lVVI2karliD94yw/UJUREqob9FfEEM+tjZn3NrC+QAfQs9lykZklIhrZHwc40mPrfoNOIiBCzn23rgH8Ve76+2HMHjPIrlEiVNWY8PNAOvr4HBo2DPRP8iYhUujKLuHNuZGUGEYkItRvCoCth2n+94Vib9Qk6kYjUYOHcJy4ixQ0MDcG6UFMIiEiwVMRFDlSDdhCfDD//DwoKgk4jIjWYirjIgTKDPufDro2w8oeg04hIDVbmNfHyeqA752ZVfByRCHHkDTD1cZjyuNdjXUQkAPvrnf5QaJkA9AfmAAb0BKYBR/gbTaQKq90IGneDZZ/D7m2QWD/oRCJSA5V5Ot05NzLUQ30l0Nc519851w/oAyyvrIAiVdbRd4ArgF++DjqJiNRQ4VwT7+Kcm1f4xDk3H+jtWyKRSNFykLecPzHYHCJSY+3vdHqhRWb2LN7sZQ44H1jkayqRSFCrAXQ4BhZ/CFk7IaFu0IlEpIYJ50j8YmAB3hzi/4c3g9nFPmYSiRy9z/OWiz8KNoeI1EjlHomHJjx5OPQQkeI6Huct37vaOyqvkxJsHhGpUco9EjezYWb2hZktNbMVhY/KCCdS5cXXgdbDwOXDxzcGnUZEaphwrok/B/wBmAnk+xtHJAJd/DE8eQQsfBfycyE6NuhEIlJDhHNNfIdz7hPn3Ebn3JbCh+/JRCLJ4Ku85ff/2v9+IiIVKJwi/o2ZPWBmQwrnEtd84iIl9DzbW076O+RlB5tFRGqMcE6nh26GpX+x1zSfuEhxUVEw9DqY/CgsfA96nhV0IhGpAcLpna55xUXCccxfYe4Er6e6iriIVIJwjsQxs5OA7njjqAPgnPubX6FEIlJUFHQ6Hma9CD8+CsOuCzqRiFRz4dxi9iQwFrgWbwKUM4HWPucSiUzH3OUtv7gddm8PMomI1ADhdGwb6pz7HbDNOfdXYAjQ0t9YIhGqVgMYcau3/tRRWEFesHlEpFoLp4jvDi0zzawZkAu09S+SSIQbcbM3ktv2lbRc/W7QaUSkGguniH9oZvWAB4BZQCrwmo+ZRCLfuROgyeG0WvWmbjkTEd+UW8Sdc3c757Y7597GuxbexTl3h//RRCKYGQy8nJj8LPjoj0GnEZFqKpwj8SLOuWzn3A6/wohUK73PY1etFvDzyzDnjaDTiEg1dEBFXEQOQHQM83qETlp9dAMUaOoBEalYKuIiPspKbALH3QM5GbB6WtBxRKSaCauIm1lzMxtqZkcVPvwOJlJt9DnfW05/LtgcIlLtlDtim5n9E2+wl4XsmYrUAd/5mEuk+kisDw076khcRCpcOMOungp0ds7pPhmRg9X5BJj8H1g7G5r1DjqNiFQT4ZxOXwHE+h1EpFobeLm3/PLOYHOISLUSThHPBGab2VNm9mjhI5wPN7MTzGyJmS03s1v2s98AM8s3szHhBheJKPVaQauhsGIS7NocdBoRqSbCKeLvA3cDk4GZxR77ZWbRwOPAaKAbcI6ZdStjv38Cn4UfWyQCjbjZW855PdgcIlJthDOf+ItmFgd0Cr20xDmXG8ZnDwSWO+dWAJjZ68ApeB3kirsWeBsYEHZqkUjU5ijA4PsHYeg1QacRkWognN7pI4AX8cZMN6ClmV3onCuvd3pzYHWx52nAoBKf3Rw4DRjFfoq4mV0OXA6QkpLCpEmTyostBygjI0Pt6oOS7dqnbieSdy5hwYR72NT4iOCCRTj9vPpD7eoPP9s1nN7pDwHHOeeWAJhZJ7wJUPqV8z4r5TVX4vm/gZudc/lmpe0eepNzTwNPA3Tu3NmNGDEijNhyICZNmoTateLt0679P4IHO9B94QMw5laI0nhLB0M/r/5Qu/rDz3YN5zdIbGEBB3DOLSW83upp7D3veAtgbYl9+gOvm1kqMAZ4wsxODeOzRSJTnRQYdKW3/s4VwWYRkYgXThGfYWbPmdmI0OMZwujYBkwHOppZ29A19bPxOskVcc61dc61cc61Ad4CrnLOvXtg34JIhDnuHm85b4ImRhGRQxJOEb8SWABcB1yP1zFtXHlvcs7lAdfg9TpfBExwzi0ws3FmVu77Raqt6Bj40wpv/Z3LoaAg2DwiErHC6Z2eDfwr9DggzrmPgY9LvPZkGftedKCfLxKxajeEzifCko/hlTFw3lu6Pi4iB6zM3xpmNiG0nGdmc0s+Ki+iSDV15osQXxd++Qpm/y/oNCISgfZ3JH59aHlyZQQRqXFi4uBPv8A9KTD/bej7u6ATiUiEKfNI3Dm3LrS6GVjtnFsJxAO92LeXuYgcjJg4aD3MG441fX3QaUQkwoRzEe47ICE0MMtXwMXAeD9DidQoI2/1lp/fHmwOEYk44RRxc85lAqcD/3HOnYY3FrqIVIQ2oZHb5k2ArJ3BZhGRiBJWETezIcB5wEeh18IZ6U1EwjUidDT+wonB5hCRiBJOEf8/4M/AO6H7vNsB3/iaSqSmGX4TJNSDDfPAlRydWESkdOUWcefct8653zrn/hl6vsI5d53/0URqEDMYEprZ7Is7gs0iIhFjf/eJ/zu0/MDM3i/5qLSEIjXFwMu85eRHIfXHYLOISETY37Xtl0PLBysjiEiNl1gPrpkJj/WDGc9Bm2FBJxKRKq7MIu6cK5zkZAaw2zlXAGBm0Xj3i4tIRWvUAXqcCfPe9IZl7TEm6EQiUoWF07HtK6BWseeJwJf+xBERRt7mLSdeBnk5wWYRkSotnCKe4JzLKHwSWq+1n/1F5FA0aAujbgdX4M05rt7qIlKGcIr4LjPrW/jEzPoBu/2LJCIc+Udo2AEWTISZLwSdRiRw2Xn5PPrVMjJz8opec87xxKTlbNyZFWCyYIV7n/ibZva9mX0PvIE3T7iI+MUMxv0A0XHw5V06Gpca74UfU/nXF0t5ZeqqotcWr0/n/k+X8NwPvwaYLFjh3Cc+HegCXAlcBXQt1ulNRPwSmwj9LoKsHbDog6DTiARq3podADSoHVf02obQEfgx3ZoEkqkqKLeIm1kt4GbgeufcPKCNmWl6UpHKcNw93tH4golBJxEJVK3YaACSEvbcVLVmu3dl97C6CYFkqgrCOZ3+ApADDAk9TwPu8S2RiOwRE+/NM77kE8jZFXQakcD884yeLL93NMcWO+pes80r4kc/9C2bM7KDihaocIp4e+fc/UAugHNuN2C+phKRPToeB3lZ8P1DQScRCUxUlBETHYXZnvKzbod3Oj0nv6DodHtNE04RzzGzRMABmFl7oGb+ySMShLbDIS7JK+Lr5wedRqTS/bIpg3Z//ojhD3zDnNXbi15/8MxefPenkQDMT1MRL8udwKdASzN7BW/wl5t8TSUie8QmwBXfeuu6b1xqoCXr0ylwsHJLJr9u3nNZKTrKaNWwFu1SajNXR+L7MrMooD5wOnAR8BrQ3zk3yfdkIrJHw/bQ4VjYMB/euiToNCKVasn69KL1gtAfsbn5Bdz6zjymrdhCj+bJzNOR+L5C46Vf45zb4pz7yDn3oXNucyVlE5HiznjWWy6YCFOeCDaLSCVauiGd2GjvWnh+gVfE1+/I4tVpq0jdsovf9GzGeYNaFW2rScI5nf6Fmd1oZi3NrEHhw/dkIrK3xHpw7Sxv/bM/ez3WRaq5ggLH/LU76Nq0LrDnalLh7WXN69XimG5NuPbojkRH1bw+1+EU8UuAq4HvgJmhxww/Q4lIGRq2h0s+99bfvVLXx6XaK3COU3s35/juhxEfE1V0b1Th7WXN6ycCsG1XDqu3ZgYVMzD7m08cAOdc28oIIiJhajUI+l/qzTm+9DPofELQiUR8ExMdxR+P6wzA1SM7FL1eeCTeNNkb6GXMk5Np26g2z144oPJDBiicEdsSzOwGM5toZm+b2f+ZWc0dHkekKhh5q7d8baymK5VqKSs3n989/xNTftlS6vZd2Xk0TU4gITSSW4/myTXyXvFwTqe/BHQH/gM8BnQDXvYzlIiUo3Yj72gc4BPd8SnVz32fLOa7pZvIzssHICM7jxvemM13SzcB8OcTu/LjzaOK9u/Roh4bdmbXuBnNwininZ1zlzrnvgk9Lgc6+R1MRMpxwj+85cwXYN2cYLOIr857dirP//ArC9fuDDpKpfhm8UbGT07l4mFtGNG5MQB5+QVM/HkNyzdmFO0XVawjW4/myQA17mg8nCL+s5kNLnxiZoOAH/2LJCJhiYmHSz7z1ue8HmwW8c3unHx+XL6Fv324kBMf/Z7UzdV7DP3NGdn86a05dG6SxM0ndCl6vbBgFzjH7NXbGfvUlL0KevdmdTGDuTXsfvFwivggYLKZpZpZKjAFGG5m88xsrq/pRGT/Wg2GLifD1Cdg1bSg04gPFq3f++j7N4/9QGZOXkBp/PfqtFXszMrjkXN6F13vBoi2PUX8y4UbmLFyG43q7JmWtHZ8DI+e3YfT+jSv9MxBCqeInwC0BYaHHm2BE4GTgd/4F01EwnLs37zlhN9pprNqaEdmLgCn9G4GQHpWHis27fl3zskr4KHPl1SbI/RrR3Xg3auG0eWwunu9HmWFg73AV4s30q91ferVittrn9/0akabRrUrLWtVUG4Rd86t3N+jMkKKyH40bO/NdJaxHp4YXP7+ElFGdmlM6n0ncWKPpkWvNSx2BDonbTv/+Xo5f3l3Pu/8nBZExArjnMPM6Nas7j7boqKgQe04tu7KZtG6nRzTtfE++2zJyGbirDS27ao5d2yEcyQuIlXdeW9CqyGwfRVsWBh0GqkgU37ZQk5eAeCNXAYwonMKTZMTi/YZ0MYbQPOH5Zv5wxtzInro0SPv/4b/fLWs1G3xMdHMuv1YWjf0jrRHdWmyzz6pW3Zxw4Q5zFy5zdecVYmKuEh1ceID3nLhu4HGkIrz1w8WcPPbXtejHi2SaZ9SmxN7NGXSko1szsjmq0Ub2J2TT+cmSUXvidRRy7ZkZJO2bTe14vc/BllKUjyn9G5G+5R9T5t3a5pMlFGjZjQrd8Q2EYkQh/WAqFj49p97BoORiDVn9XaWbEhnZBfvtHGL+rX46o8j+Mcni7h14jziY6LYlZNP7bjovW61WrYxIyKvCy/d4PU079SkTpn7XPHyDE7s0ZRHzu5T6vbEuGg6Nk5ifg0q4joSF6lOup/mLX/9LtgcEWTaii18OHdt0DH28eWiDRhw1Yj2e71+0dA25BU4duV4g6DUjo/hz6O7clJP75p58duuIsmyjd50ox0bJ5W5z2cLNpR7qvzw5snMTduBqyHzCqiIi1QnR9/hLWvoVKV5+QXc/u58Fq8Pf1CUsU9P5ZpXf2bh2p38ccIclqxP573Za7zOU//6lja3fBTIdebpqVtpn1KHpITYvV5vmpzI0V32dOr63ZDWnDuoFY+f25dmyQnMXxuZR6FLN6STlBBDk7rx+93vpSkrycsvKHN7j+Z12ZyRzcb07IqOWCXpdLpIwDbuzKJRnXjyChzZefk8/MUy4mKiuHBoa/7z9XJenbaKL28YTofGdcjLL+DLRRt47Jvl9G/dgO7N6vKnt+Zy3+k9GDugJVavJSS3gl+/hW2pUL9NheUs7DlcUOBYsiG9aGrInLwC0rNyaVhn/798S9qSkU1MVBQ5+QX8vGobR3dtUu5Ukss3ZhTNK11avq8Xb+TlqSt5eepKzh/cipkrt/O3U7rz7ZJNHN21MX1a1S/af+uuHKb8soVmyQms3ZHFiY9+D8Dbs/b08G5ez+tA9v2yTfRqUY/6tePYnZNPYlw0M1K30rBOPG19OHU9bcUWpq7YutdgJ8U9fHZvZqRuJTkxlr7FvqdbTuxKk6QD+3eoKnq1qEfdhFjMyp9ONCa67OPPU/s0Z3SPpjSpWzOm+LBIO+XQuXNnt2TJkqBjVDuTJk1ixIgRQceospxz7Nidy+vTV9O6QS2O734Ymbn5xEQZCbHRrN6aSVJCDKu2ZjL+x1TqJsYyfnIqAK/8fhDnPbtnIJbFd3uzjr05YzW3v7cAgFN7NyMzJ5/PF24o2u+FiwZw8fjpRc8/vu7IokJTmraNanPJEW0ZkrCSDu+GhnC4dS3ElV9knvvhV/q0qldUEPLyC+h+52ckJcSSV1DA9tC9ym0a1iJ1i9dxqnOTJJZsSC/62rXiolmwdicTrhjCWU9N4ZiujXn2wgHkFziio4wdu3O5+8OF3H5yN256aw6fLdiwV4YuhyXx6f8dtddrOzJzSd2yi5YNapGUEEPH27w51J89rhaNO/WhR/Pkol/6//hkEU99u6LM77EwD8DOrFx63uVN6frdn0Zy1APflPqewkk1BrVtwLRftxYtp/75aAb/4ysAjuzYiL+f1oOWDWqV2877My9tB50Oq0N8jDfASVZuPtFRRux+ClZFq+q/ByZMX03vVvXo1KTsU+5VUUW0q5nNdM71L/m6jsSlWlu+MZ1ZK7fz+cINxMdG8cjY3kxP3cYjXy1l6oqtRfs9PLYXufmOm97yegInxkazOzef728aydrtuxn79NS9PvenW49m4N+/2uu1Cwa35sO5a9kWKniFckqc+vvbhwvp16p+UQEHQsVy7z+oR3ROYd5dx9EjVGz+9NYcWtRPJG3bbm4Z3YVZK7fRon4tnv/xVwDOHdiK29+dD8Aj9U7hlKz3cONPYuHJ71G/Vhy/bt5FzxbJRadnt+7Koe/dX+z1NQe1bcC4Ee25+AXvj4fsjL1PSW4qdoqysIAD/FpsoJGznpoCwJeLNtLmlo8o6a2Ze450rziqHdszc3ljxmoWr09n484sGoeOoP7+8SKe/m5PUX7y/L6A186//3wlfP4j/zyjB8s3ZnDViA50Ocz7xd67ZT1O6d2Mv36wkOgoKzoV/uWijXT+yydM/8sx/G/qniEuWjWsRep9J7E9M4dacTHERBm7c/N57adVnN63BX3v/oJpv3o/K4XLwgIO8P2yzRx5/zf0aVWPly8dRJ1yeleXZs323fzmsR9o07AWz/yuP83qJVL7ID7HOce0X7cSGx1Fv9b1y39DFbFux24SYqKpXztuv/udNaBlWJ83cVYaG3Zmc2WJ/gTVkYq4RCznHLe9O5+cvAJa1E8kPSuP7s3q0rWp97jl7bm8Pn31Xu+5cnh7WtRP3KuAAxzRIYUvih0F7871Og39vHo7KSVOE189sj31a8cxsG0Dfgr9Uk+Kj+GK4e1ol1Kb92av5aYTOjOkXUO+/fZbRnT2BuvIyM6j/z1fcOLhTcnNL+Dknk0Z2r4R5wxsWXQ0+di5e3+PSQmxpN53EjuzcqkTF7NXL+RCd/ymW9H6vR8vAuD67WMZmDSLpmt/ZsLjt/Ni/vFF+5h5R9GL1+8pwsd2a8IXCzcw7detRYUKYMZfjmHllkzO+O9kvv7jcFo1qMVT361gcLuGtKifyKC/f8VZ/Vswbnh7xjw5ha27chjcrgFRZkwuYwrJQsvvHV10WnTemh0sXLeTgX//inHD27N6ayYfzVu31/5PfruCerVieblYAX7ky2Ws3ZHFmzPTmH3HcZzWp0XRtnMGtiIhNppd2XksXLeTM5+cQnZeAf3u/oLeLesBMO3Wo4v2Lz76V+34GH5/ZDsAhrRryNodu1m5JZOUpPi9/pDp1KROUa/qn1dt58uFGzg1zGE/CwocnW//hBGdG3NGXy936pZMjn34O9o1qs1bVw6lQTlFrSQz45a359KxSRLP/G6fg7aDMvapKRzZsRHXjOpYIZ9Xmke+XMYn89cz6/Zjy72kEo7Jv2xh0pKNjBveLqzT85FMRVwqTVZuPgXOkRgbTUZ2HjFRUSTGRbNm+25+XLaZXTl5RJnx6+ZdDOvQiGO7NSEvv4D/TV3Jll05LNuQwacL1gPw6f8dSbtGdXh12qp9vs7lR7Wja9O6NAoV3+GdUji1TzOGdWhE4yTvKC/1vpP2ed9Z/VtwWp/m5BUUMDdtB0PbNyz6BVDa/hOuGEJefgErt2bSpmFtoqOMi4e15eJhbUv9/uvEx7D47tFFz0d22XfEqbLULdG5qSyp953EOz+nce9Hizg+/Q7mJlzGxdGfsrz1WH5csR0A5ygq4Ded0Jkz+ragSd0Evl26iUvHT+fNcUN4f85arhvVkfq142hUJ36v7//qkR32+nqFZt1+7F5ZNqVns2xDOh0a1yEmOopRD03iifP6MrR9o31yf3z9kUVH7U9++0vR69/fNJKfV3vF8f05a3l4bC8Oq5vID9N/5k9nH8MFz01j7Y4stmfmsnLLrqKBQICicbdrx8fQv3V9XrxkIBc+/xO5+Y7+bRpw+8ndwrpu+trl3ih4n8xbx+7cfBJio7nqlVm0bJDI+9ccweRfNvPIV8uZs3o7hyUnsGDtDro3Sy73c3dm5ZKb7/hi4YaiPyBP7d2Md2evpcA56tcK79+8pFYNa1fodJzTft1KVl6Bb0XcOcc3SzYyrEPDCingAD1bJPPWzDTW78zaa2Cc6khFvJis3HwWrN2JGRjeWL0Or3NLSlI8mTl5RVMBOrxfhs452qbUpnFSAjuzcpmXtsN7HUdhd4MuTZOKtq/YtIso2zMOcJ34GJrUTSAxLprdOfls2ZVd9D7nIDraaFQnjviYaDJz8sjIyqNerTiio2y/P/DOObJyC0jPzi0qXIvX72TFpl0Y3tFYbHQU0VFWNNXfk9/+wpzV20mMiyYhNprcvALaptTmqhEd2JSezf2fLiYjO49pv27lmK6NqR0fwzkDW9GpSRJTftnCo18tIyM7j3lrdpAQG0W7RnV4+dKBZObkc82rs5i3ZgfFzxhfOaI9N5/QhXlp27np7b3n0nntp1Us+tsJREcZ9368iNx8R3Ki90utbaPapGflERcTxfTbjmHhup0kJ8aybvtu2jeuU9QZ6cbjO3Pj8Z3D/vePiY7CuxwZzbAO+xaast7TPqXs+1qDcFqfFnuOSGfn0+bdcbyS9ye4bzL5BY5N6dkclrxv8RreKYXlfz8RYK8OYAcrJSmelGKdrGbfcdx+9x83vP1eBXzh346nVlwMLRvUokNKHXq2SObU3s0xM7JXe7+6nr9oAB/PW8dPv27db0E2M+/7u3c0ufmOxLjoMvcty+hiw57O+MsxFBQ4EmKjGdWlCaO6NOH9OWu5870FLNmQzpQ/j9qneOzMyuW2d+YzsG0DejRPpnfLenRsXIdlGzOKrrWf0a8F785eS7/WDQ76CLJOfDRrt+fjnOPvHy/i5J7N6BU683CgCkeLO7aUIU4rysJ1O9mwM5uRnSvuaxwempZ0btoOFfGaZFN6Nmf8d/I+r9/1m25cNKwtq7ZmMubJKftsf2BMT87s35JlG9L36sBU6L/n9WV0j6b8vGo7Fz7/0z7bX750IEd2TOGrxRu45tWf99n+3tXD6NWyHhNnreEvoWueALHRRlx0FFNuPZq6CbH864ulvDQlldy8ArLzCsgrcJjBL/eeSFSU8eLkVF77ae/Ty3XiY5h7p/fLdd323Xy/bDMZ2XnUqxVLtBn9s7xf5j8s38SbM9NompxAfoFjwow0khJiGN4phU5Nkli4biertmZyWHICx3RtzIpNu2hSN54GteOIjsqlfu04LhnWltXbMr2BKBwc09UbNvHIjil8/oejSE6MZefuXD6et57f9m7m/TFlxk+3HkPdxNhS/2hJSYpneFIKQNEpUimm51j48A+wcQEs+pDorieXWsCrgltGd+GW0V5v7Nz8gr06dHVrVrfU8bRjo6M4pXdzTukd3insPX+oHZpGpfTEP7ZrE657zfv/O+QfX/PZ/x1F58P2dMD6cuEGFqzZwQdzvHvSF999Av89vy+fL9zAlcPbk5VbQEJsFPef0ZNRh1A0a8XFkJmdR0Z2Hs98/yvPfP9rqWeSwrE1NAb5r5v9GwVu0pJNAAzvnFJhn9mtaV2io4z5a3ZwfPfDKuxzqyIV8WJSkuIZf/EAHIDzprwz2zP4QIv6tXjpkoEU/oFsWGi7dyTWqUkSb1w+GDMrOpo3g3aNvO2HN6vLCxcNoMA58gscBQ4yc/KKhkzs1aIe95/RE4rea+TmF9C8vveX5OB2Dbj71MPZnJ7Njt255OQXkBgbTWFpO7xZXX7TsxlxMVHEx0SRlBBLnYQYCpwjCuOqER04f3BroswocI7cfEetYkckfz3lcP56yuGlts1eR3eluPSItlx6ROmnkevVimP8xQPLfG/t+Jii3qZN6iZwfYmep+V1dpH9iIqCq6fBIz3hp6eg68lBJwpLZfbIriglj+7/8cmivX7u1+3IYkWxDoCb0rPp0DiJDqHfL4XvD7fzVlnqxMewdkcWXy/eWPRaXn7Bfm/LKsua7bsB77a7B8/s6cv15e+XbaJb07pFZwwrQkJsNN2a1mXH7tzyd45wKuLFJMRGF51aLk2d+BiO6lT2X4tJCbEMatewzO0N68Tv9zpoywa19nubSvH/8KU5rvthHLefvzoP9RYYiVD1W8ORN8L3D8LcCdDzrKATVVuFnRB73vU5k5Zs4odlmzmiYyNe/2kV4yen0jQ5geGdUoiJNt/+P156RFtembaSp75dwRl9W/D2rDRmr95O/9BEKQdrV07+QfW8L88/Tu/J1l0VPzDLe1cPK7UjaHUTeX/uisiBG/FnsGiYeBnkVlynJ9lX8U6I5z83jQuf/4lbJs6jcVI8D4/tzX1n9OSeU3v49vVbNqjFpUe0Y+mGdG4Z3YWPrjuCGSu3cfeHBz67Xb/W9fnXWb2APbcXrt+RRe5+Rkw7UG0b1aZf60P7A6M0NaGAg4q4SM0QHQPH/s1bf/UsiLBBniLNF384ijYNvSPtb5d613xfuGgAg/dzpq4iDW3fkLwCx7M/rCClTjz3fbKY5374tdz3bc8uYGP63n/kFXZO3JSezTs/p3HEP7/mxjfnVEjO9+es5b3Zayrks0pat2M3Zz89ha8Xbyh/5wimIi5SUwy9Bjqd4A3J+s29Qaep1jo2SeKVywYzqG0Dnr+oPzP/ckzRIDaVoX8br0PqU9+uICHMnvg5eQX8b2EOA+/9ilemeffiPzFpORc853XGff6HX/nDG3OoVyuO92av5bvQHyeH4sXJqXsNvFOR6teKY0bqNmakVu+5xX0t4mZ2gpktMbPlZnZLKdvPM7O5ocdkM+vlZx6RGu/M8RBbC757ABbvO5qaVJzm9RJ544ohjOrS5IDHlT9UteJiuP+Mnrx95VDqJsRyep/mtGxQ9q1Wyzemc+rjPzJjgzfI0ctTvMK6bnsW8TFR/Pe8vtx4fCeuHdWBb24cTttGtXnkq2WHnHP9jixa1Penb0BCbDQdmyQxr5pPS+pbxzYziwYeB44F0oDpZva+c674hZlfgeHOuW1mNhp4GhjkVyaRGi82Ea6dBf/qAq+fC0f8AY65K+hU4oPivdy7Nq1LVJTx6fx15BU4Tu7ZrGjbzJXbOPeZqdSOj+G6PvGsphEzVm5lR2Yu01O30rxeYtE98n88zht34cnz+3HYIZ5ZcM4bs6BxObOWHYqezZP5bOH6osl7qiM/j8QHAsudcyuccznA68ApxXdwzk12zhWe65gKlH0Pk4hUjLpN4WJvIhF+eBg+uTnYPOK7y45qR3JiLOP+N4tHvtz7CHrqii3k5Bfw6fVH0rdJDLXjo1m9dTfnPDOVXzZlFN27X1znw5JIrhVLXn5B0b3kB2p7pnebbEXeWlbS4S2S2Z6ZS9q23b59jaD5WcSbA8VHFkkLvVaWS4FPfMwjIoVaD4U/LoWoGJj2JLx9WdCJxGdNkxOoVyu2aF4A8I6Grx7ZgcV3n1B0zb5w4pUtu7J59sIBZd626pzjgud+4trXZnEws2Fu2ZWDGTT2cerUgW0acHSXxkWz8FVHvk1FamZnAsc7534fen4BMNA5d20p+44EngCOcM7tM2uCmV0OXA6QkpLSb8KECb5krskyMjKoU6dqDR9aHVT1do3N2UnfWX8iMWs9iztfx/qmR5f/piqgqrdrVTV+QTazNuTzrxGJTFyWS3qO49Iee4poRkYGX2+IY+KyXJ4+thZxZczdXujrVbm8tDCHy3vGM7TZgV+dzcx1RBkkxFTPU92FKuLndeTIkaVORepnER8C3OWcOz70/M8Azrl/lNivJ/AOMNo5t7S8z9V84v6o6vMIR6qIaNfsDHi4uzfv+P/Ng6gKGJfUZxHRrlXQ6z+t4uvFG8nIzmPyL1sY278lfz+9R9GQxpMmTaLv4GFk5eaTUie+3OvIBQWO0/87mbRtmXx1wwiSD3LSlurOz/nE/TydPh3oaGZtzSwOOBt4v0SoVsBE4IJwCriI+CC+Dpz8MOxcA29cAAUVN5CHVC1n9W/Jmu27mZ66lX+d1Yt/jum5z5wEdRNiaZyUEFZHsKgo497TDmfrrhzu/2zxAWX5y7vzePdnf+4RL5SelcuAe7/ca/ra6sa3Iu6cywOuAT4DFgETnHMLzGycmY0L7XYH0BB4wsxmm9kMv/KIyH4cfjr0vRCWfARvnK/BYKqpn1dvZ8Handx9yuGc3rdi+hF3b5bMxcPaMiN1G1nFrreXxTnHo18t439TV7F2h78dzmKjo9iUns3OajyGuq9jpzvnPgY+LvHak8XWfw/83s8MIhKm3zwC21d6hfzD//OeS7Wycos3AUtFz+x143GdufmELsTF7P+4sKDA8dcPFvDilJWc3rc5lx3ZrkJzlBQfE4UZZIfxx0Wk0ohtIuIx8waDqdMEZo6HF06CVVODTiUV6PS+LVh27+gKnxkwMS6auJgo8vILyC8o/SxOfoHj+jdm8+KUlVx2ZFseHNPL99nqzIz4mCiy8qrvJSIVcRHZI7E+/N98aDUEVv4Azx8Pu/a5YUQimF+F8+vFG+j0l09YtG5nqdujo4zm9RL58+gu3HZSt0qboCQhNlpH4iJSg8TEwSWfwmlPe88faA9rZgabSaq8+rXiKHDsNYFKVm4+932ymMm/bAbgltFduGJ4+0rNdXLPpnRvnlypX7MyaT5xESldr7GweQl8/xB8fS9cMDHoRFKFNQkNFrNhpzdl6eRfNnPrxHmkbskkLiaKoe0bBZLLz2lfqwIdiYtI2Y6+A7qfBr98BVMeh7yDG2JTqr+UpHjMYMPOLFZtyeSC537CAa/+fhA3HNsp6HjVloq4iOzfsOu95We3wgujdfuZlCo2OoqGtePZsDOLTxesI7/A8b9LBzG0QzBH4IXGPjWF3784PdAMflIRF5H9a9YHblkNnUbDmhnw755BJ5Iq6uJhbRjWoRGJcTEc370JLRv4M83ogXAOMrLzgo7hG10TF5HyJdSFM571OrntWAUPHw5XfAe1GgSdTKqQq0d2KFq/YHDrAJPsER8bVa2LuI7ERSQ88XXgz2tg4OWwYzV8+AedWpe95OYXsH5HVvk7VqL4mGiycnWfuIgIRMfAiQ94Hd4WvgsPdoT86nuUIwfmsa+XM/gfX9HpL5/w1aINQccBICE2SveJi4jsZeh1kNwKdm2CiRo5WTyN63rTmubkFTAs4A5thY7qlMLJvZoFHcM3uiYuIgcuOhaun+1dG1/wDuTuhrNfgygdF9Rkjep4RfzG4zqREFs1prQ9q3/LoCP4Sv/jROTgREXDZV9Dx+Ng6afw0Q1BJ5KAHdO1Ca/+ftBeHdyC5pwjR2Oni4iUom5TOOtlb33mC/DibyCr9LGzpfqLjjKGdmgU1lzkleXBz5fQ7Y5Pg47hGxVxETk0sQnwxyXQpAf8+h28fm7QiUSKxMdEk1fgyMuvnkfjKuIicuiSDoMrf4CYREj9Ht65EjYtDTqVCAmxXpmrrtORqoiLSMW55BNISIY5r8KrZ0JB9b21RyJDfIzXwa663mamIi4iFadZH7hlFfS5ALalwvRng04kNZyOxEVEDtRJD3nLT26Cqf/1bkETCUC3pslcOaI9tarILW8VTUVcRCpeTDxc+AEk1INPb4H/9Ifd24NOJTVQjxbJ3HxCF+rXjgs6ii9UxEXEH22PgptWQO/zYWcaPHUU5GUHnUpqmLz8ArZn5lTbe8VVxEXEP1HRcMpj0O9i2L4SnhoOuVVrggyp3qanbqP3375gxsqtQUfxhYq4iPjLDE5+GNqPgk2L4IlBkLEp6FRSQxR2bMuupjOZqYiLiP/M4PyJ0OVkr9f6gx3gv0fA0s80C5r4qugWszzdYiYicvDM4OxX4NwJ0GIAbJgHr54F/2wDC94NOp1UU0W3mOlIXESkAnQ6Hn7/JVwzA0beBq4A3rwQFr4fdDKphuJDt5ZlabAXEZEK1KgjDL8J/rgIomLhnXGwa3PQqaSaqZcYyx+P7cThzZODjuILFXERCVZCMoz9H+TuglfO1G1oUqFqx8dw7dEdVcRFRHzT+QRo3B3WzoKXTtEIb1Kh1mzfzbZdOUHH8IWKuIhUDb97D1oNhVVT4N7DvHvK37sGdm8LOplEuOH3f8Mz368IOoYvVMRFpGqokwIXfwwn/Qua9oJ1s+Hnl+HF32rIVjkkCbHR6p0uIuI7MxhwKVzxHdy5HUb+BdbPhX+2hh/+HXQ6iVAJsVFk6T5xEZFKZAbD/wQnPgiJ9eHLO+HBTjDxCqLyNXSrhC8+JlojtomIBGLgZXD5JOh/KdRtBnNf58jvz4FvH9BobxKWeB2Ji4gEqH4bOPlfXjE/6iaMAvjmHniwI/z0DOTnBp1QqrDrRnXkzH4tgo7hi5igA4iIHJBRt/EtAxleJxW+/Ct8fCN8cy90/Q0kNvDGZ2/R3zsdLwKc2qd50BF8oyNxEYk4LirWO81+41LodS7EJMDSz+HHR+C5Y+CZkfDz/3S6XQBI25bJik0ZQcfwhY7ERSRyxdWC0/6753n6eq8X+7T/wntXw8d/gi4nwW8egbjagcWUYN3+7ny27Mrh/WuOCDpKhdORuIhUH0mHwej74E+/wLF/g9xMmPcm/L2Z1xFOaqT4mGhNgCIiEjFqN4Jh18NNv3r3mickex3h7m8P056CuW/CxsXgXNBJpRIkxEaRnVc9bzHT6XQRqb5qNfDuNR90OXx7vzfd6Sc37b1PbC3ocAyc/gzEJgSTU3zljdhWPY/EVcRFpPpLSIbj74Vj74Ztv8K6ObB6GkTFwNLPYNH7cG9oPvOxr0CnEyBavx6ri/iYqGo77Kp+SkWk5oiKgobtvcfhp3uvHX8vLP8SPvgD7FgFb5znvZ7cCpr3gWZ9YOh1EBUdXG45JKf1bUG/Ng2CjuELFXERkQ7HwB/mwc51MOdVb/CY5V/Bog9g4XveNfTDenid5ZKaBJ1WDlDvlvXo3bJe0DF8oSIuIlKoblM48o/e+ohbvI5v3/wdvrsfNi6ABe9ASmcYei20HuYNA6tBZaq8DTuzSN28i/5tGhAdVb3+vdQ7XUSkLGYw6ja4Yxtc8T30uxCydsDEy+DhbvBob3hnHKT+CAXV85prdfDBnLWMfXoqGdnVb/AfHYmLiJQnKgqa9oSmD8Dx/4Bln8GamfDr9zDndZjzGkTFQpPu3lSq3U+D+KSgU0tIQqzXnyE7Nx8SYwNOU7FUxEVEDkR0jDcKXJeTvOc718HiD2H2q7B2Frx/rffo+ztoPwrqtfZOwcckevtH6QRoZSss4tWxh7qKuIjIoajb1BvHfeBlkLMLFn8EU5+An1+BWS+V2NmgUUevk1xM6J70Bu28W+AadoBGnXSd3QfxMd4fTtnVcDpSFXERkYoSVxt6nuU9snbCmhneeO7bV3vb87Nhw0LvVHxBPhTkwexXSnxGUqgHfPFC7sCivVvjhl4HLQboPvYDoCNxERE5MAl1vdPp5cna6Y3xvmkxbFkOm5bArs1777P6J9iZBpuXwJKPvdfikyG5OdRv652iT+kCdZp407A27g4xcRX/PUWoXi2TeeqCfrRqUCvoKBXO1yJuZicAjwDRwLPOuftKbLfQ9hOBTOAi59wsPzOJiFQpCXW9R9Jh0G5E2fs55400t24OpK/zCv3an2H7Su8PgUUf7Nk3Ot47TV+nMdRv412Tr9vcO02f1BRqp5QeZfd62Pqrd0o/Jr5Cv80gNU5K4PjuhwUdwxe+FXEziwYeB44F0oDpZva+c25hsd1GAx1Dj0HAf0NLEREpzgya9fYepSnIh+2rIG06rJwMuzZ5p/IXfwizXgzrSwwGmIbX0z622FFrcgtvHPpCUTFekY+J967tR8d5y4S63pkBv/4AiI7z+g5ElShd0bGhR1yJRyyYkZGdx/Rft9K9WV0a161e4+P7eSQ+EFjunFsBYGavA6cAxYv4KcBLzjkHTDWzembW1Dm3zsdcIiLVT1Q0NGjrPXqetfe2XZshYwPkZsHWX7yCX4pFixfRtWN72Lx0zz6uwBtvPmdX6LmDvGzI3OIt87IgP8dbZqd71/mrkuh44hPq0yg9gU0x8WwLDfYSE220T6kDQNq23ewqcQ95XEwUbRt5c9Cv2prJ7py92ywhNprWDb0/dFK37CK72PX2RknxNDz7SWjSzbdvq5CfRbw5sLrY8zT2PcoubZ/mwF5F3MwuBy4HSElJYdKkSRWdtcbLyMhQu/pA7eoPteuhaFzmlow6SWxIrwPxrffeEOalZCvIIz57M+b86UAWn72Z2NydJV51mMsnqiCPqIJczOURVZCHudzQazlE5+wgL3srzhWQHZp9Njoftu7KBSA9L5rdbu9b/3Lyrdj2GLLd3mPn5xbbnpEXS06xaW3jsuGXGTPJrL3R2+7jz6ufRby0eyRKTt4bzj44554Gngbo3LmzGzFixCGHk71NmjQJtWvFU7v6Q+3qj+rcrs33s628qVEOZnvbYut+tqufow6kAS2LPW8BrD2IfURERKQUfhbx6UBHM2trZnHA2cD7JfZ5H/ideQYDO3Q9XEREJDy+nU53zuWZ2TXAZ3i3mD3vnFtgZuNC258EPsa7vWw53i1mF/uVR0REpLrx9T5x59zHeIW6+GtPFlt3wNV+ZhAREamuNBK/iIhIhFIRFxERiVAq4iIiIhFKRVxERCRCqYiLiIhEKBVxERGRCKUiLiIiEqFUxEVERCKUiriIiEiEMuf2mTSsSjOzdGBJ0DmqoUbA5qBDVENqV3+oXf2hdvVHRbRra+dcSskXfR121SdLnHP9gw5R3ZjZDLVrxVO7+kPt6g+1qz/8bFedThcREYlQKuIiIiIRKhKL+NNBB6im1K7+ULv6Q+3qD7WrP3xr14jr2CYiIiKeSDwSFxEREapYETezE8xsiZktN7NbStluZvZoaPtcM+tbbFuqmc0zs9lmNqNyk1dtYbRrFzObYmbZZnZjiW1q11KE0abnhX5G55rZZDPrVWyb2rQMYbTrKaE2nW1mM8zsiGLb1K5lKK9di+03wMzyzWxMsdfUrmUI4+d1hJntCLXdbDO7o9i2imlX51yVeADRwC9AOyAOmAN0K7HPicAngAGDgWnFtqUCjYL+PqraI8x2bQwMAO4FbiyxTe16cG06FKgfWh+tn9UKa9c67LkM2BNYrHY99HYttt/XwMfAGLXrobcrMAL4sIz3V0i7VqUj8YHAcufcCudcDvA6cEqJfU4BXnKeqUA9M2ta2UEjTLnt6pzb6JybDuQGETAChdOmk51z20JPpwItKjljJAqnXTNc6DcgUBtQp57yhfO7FeBa4G1gY2WGi2DhtquvqlIRbw6sLvY8LfRauPs44HMzm2lml/uWMvKE0677o3bd14G26aV4Z5AKqU1LF1a7mtlpZrYY+Ai4pNgmtWvpym1XM2sOnAY8Wcr71a6lC/f3wBAzm2Nmn5hZ92KvV0i7VqUR26yU10r+lb2/fYY559aaWWPgCzNb7Jz7rkITRqZw2nV/1K77CrtNzWwkXhE/otjLatPShdWuzrl3gHfM7CjgbuCY0Ca1a+nCadd/Azc75/LN9tld7Vq6cNp1Ft5wqRlmdiLwLtAxtK1C2rUqHYmnAS2LPW8BrA13H+dc4XIj8A7eqQ4Jr13LpHYtVVhtamY9gWeBU5xzWwpfV5uW6YB+VkO/8NqbWaPQc7Vr6cJp1/7A62aWCowBnjCzU0Htuh/ltqtzbqdzLiO0/jEQW9E/r1WpiE8HOppZWzOLA84G3i+xz/vA70K91AcDO5xz68ystpklAZhZbeA4YH5lhq/CwmnXUqldy1Rum5pZK2AicIFzbmmx19WmZQunXTtY6FDRvLtT4oAtatf9KrddnXNtnXNtnHNtgLeAq5xz76pd9yucn9fDiv28DsSruRX681plTqc75/LM7BrgM7xef8875xaY2bjQ9ifxek2eCCwHMoGLQ29vgnd6Dbzv6VXn3KeV/C1USeG0q5kdBswA6gIFZvZ/QDe8mXfUriWE+bN6B9AQ74gGIM95EyDoZ7UMYbbrGXh/yOcCu4GxzjlnZmrXMoTZrmVRu5YhzHYdA1xpZnl4P69nV/TPq0ZsExERiVBV6XS6iIiIHAAVcRERkQilIi4iIhKhVMRFREQilIq4iIhIhFIRF6kizKxhsdmO1pvZmtD6djNb6MPXu8tKzFoXxnsyynh9vBWb+eoQMlXI54jUFCriIlWEc26Lc663c6433hjWD4fWewMF5b3fzKrMuA8iUjlUxEUiQ7SZPWNmC8zsczNLBDCzSWb2dzP7FrjezPqZ2behSRU+s9Asf2Z2nZktNG8u7teLfW630GesMLPrCl80sxvMbH7o8X8lw4RGTXws9Jkf4U1nW3Kfrmb2U7Hnbcxsbmj9DjObHvr8pwtHtSrx/lQLDVFpZv3NbFJovbaZPR96/89mVukzR4lUFSriIpGhI/C4c647sB1v5LJC9Zxzw4FHgf/gzQXdD3geb454gFuAPs65nsC4Yu/tAhyPN27znWYWa2b98EZDHAQMBi4zsz4l8pwGdAZ6AJfhzZ++F+fcIiDOzNqFXhoLTAitP+acG+CcOxxIBE4+gLa4DfjaOTcAGAk8EBq6UqTGUREXiQy/Oudmh9ZnAm2KbXsjtOwMHI43I9Js4C/smcd8LvCKmZ0P5BV770fOuWzn3Ga8eaSb4M249o5zbldo8oaJwJEl8hwFvOacyw9N5PB1GbknAGeF1scWyzrSzKaZ2TxgFNC9tDeX4TjgltD3OAlIAFodwPtFqg1dQxOJDNnF1vPxjl4L7QotDVjgnBtSyvtPwiu8vwVutz3zGpf83BhKn2KxNOGM2fwG8KaZTQScc26ZmSUATwD9nXOrzewuvEJcUh57DjSKbzfgDOfckjBzilRbOhIXqT6WAClmNgQgdGq8u5lFAS2dc98ANwH1gDr7+ZzvgFPNrFboNPVpwPel7HO2mUWHrruPLO2DnHO/4P1xcDt7jsILC/JmM6uDN0lEaVKBfqH14pcPPgOuLTY7VMlT/SI1ho7ERaoJ51xO6PasR80sGe//97+BpcD/Qq8ZXq/37aX0JSv8nFlmNh4o7JT2rHPu5xK7vYN3Gnxe6PO/3U+0N4AHgLahz99uZs+E3puKN6Vjaf4KPGdmtwLTir1+d+j7mhsq5Kkc2DV1kWpDs5iJiIhEKJ1OFxERiVAq4iIiIhFKRVxERCRCqYiLiIhEKBVxERGRCKUiLiIiEqFUxEVERCKUiriIiEiE+n/5ieVm0xywOgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 576x432 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "from sklearn.metrics import precision_recall_curve\n",
    "\n",
    "def precision_recall_curve_plot(y_test, pred_proba):\n",
    "    precisions, recalls, thresholds = precision_recall_curve(y_test, pred_proba)\n",
    "    \n",
    "    plt.figure(figsize = (8,6))\n",
    "    # x축 범위 설정\n",
    "    threshold_boundary = thresholds.shape[0]\n",
    "    \n",
    "    #정밀도와 재현율 시각화\n",
    "    plt.plot(thresholds, precisions[0:threshold_boundary], linestyle='--', label = 'precision')\n",
    "    plt.plot(thresholds, recalls[0:threshold_boundary], label='recall')\n",
    "    \n",
    "    start,end =plt.xlim()\n",
    "    # x축 설정\n",
    "    plt.xticks(np.round(np.arange(start,end,0.1),2))\n",
    "    \n",
    "    plt.xlabel('Threshold value');\n",
    "    plt.ylabel('precision and Recall value')\n",
    "    plt.legend();\n",
    "    plt.grid()\n",
    "    plt.show()\n",
    "    \n",
    "precision_recall_curve_plot(y_test, lr_clf.predict_proba(X_test)[:,1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b1c6cdbf",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}