TEST_ML-Copy7.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pathlib import Path\n",
    "from datetime import datetime\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn import preprocessing\n",
    "pd.set_option(\"display.max_columns\", 100)\n",
    "from sklearn.metrics import f1_score\n",
    "DATA_PATH = Path.cwd().parent / \"ML project\" \n",
    "\n",
    "from sklearn.preprocessing import PolynomialFeatures\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler, OneHotEncoder\n",
    "from sklearn.impute import SimpleImputer\n",
    "from sklearn.compose import ColumnTransformer\n",
    "\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.metrics import roc_curve, roc_auc_score\n",
    "from sklearn.preprocessing import Normalizer\n",
    "RANDOM_SEED = 6    # Set a random seed for reproducibility!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "from math import cos, asin, sqrt, pi\n",
    "\n",
    "def distanceLong(lat1, lon1, lat2, lon2):\n",
    "    p = pi/180\n",
    "    latDiff = (lat2-lat1)*p\n",
    "    lonDiff = (lon2-lon1)*p\n",
    "    coslat1 = np.cos(lat1*p)\n",
    "    coslat2 = np.cos(lat2*p)\n",
    "    coslondiff = np.cos(lonDiff)\n",
    "    coslatdiff = np.cos(latDiff)\n",
    "    a = 0.5 - coslatdiff/2 + coslat1 * coslat2 * (1-coslondiff)/2\n",
    "    return 12742 * (np.arcsin(a**0.5)) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "def formatdata(dataFrame):\n",
    "    dataFrame['pickup_time'] = dataFrame['pickup_time']+ '.099'\n",
    "    dataFrame['pickup_time'] = dataFrame['pickup_time'].apply(lambda x:  x.replace(\"/\",\"-\"))\n",
    "    dataFrame['drop_time'] = dataFrame['drop_time']+ '.099'\n",
    "\n",
    "    pd.to_datetime(dataFrame['pickup_time'], format='%m-%d-%Y %H:%M:%S.%f', errors='ignore')\n",
    "    pd.to_datetime(dataFrame['drop_time'], format='%m/%d/%Y %H:%M:%S.%f', errors='ignore')\n",
    "    \n",
    "    dataFrame['pickup_time'] = dataFrame['pickup_time'].apply(lambda x: datetime.timestamp(datetime.strptime(x,'%m-%d-%Y %H:%M.%f')))\n",
    "    dataFrame['drop_time'] = dataFrame['drop_time'].apply(lambda x: datetime.timestamp(datetime.strptime(x,'%m/%d/%Y %H:%M.%f')))\n",
    "\n",
    "    dataFrame['timediff'] =  dataFrame['drop_time'] - dataFrame['pickup_time']\n",
    "   # dataFrame['timediff1'] =  dataFrame['timediff']**2\n",
    "    #dataFrame['distance'] = distanceLong(dataFrame['drop_lat'],dataFrame['drop_lon'],dataFrame['pick_lat'],dataFrame['pick_lon'])*100\n",
    "    dataFrame['distance'] = ((dataFrame['drop_lat']-dataFrame['pick_lat'])**2+(dataFrame['drop_lon']-dataFrame['pick_lon'])**2)**0.5\n",
    "    #dataFrame['unit_meter_waiting_fair'] = dataFrame['meter_waiting_fare'] / dataFrame['meter_waiting']\n",
    "    #print(dataFrame['unit_meter_waiting_fair'])\n",
    "    \n",
    "    #dataFrame['_fare'] = dataFrame['fare'] - dataFrame['additional_fare'] \n",
    "    #dataFrame['tripDuration'] = dataFrame['duration'] - dataFrame['meter_waiting']\n",
    "    \n",
    "    \n",
    "    return dataFrame\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\User\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n",
      "C:\\Users\\User\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:3: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  This is separate from the ipykernel package so we can avoid doing imports until\n",
      "C:\\Users\\User\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  after removing the cwd from sys.path.\n",
      "C:\\Users\\User\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:9: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  if __name__ == '__main__':\n",
      "C:\\Users\\User\\anaconda3\\lib\\site-packages\\ipykernel_launcher.py:10: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  # Remove the CWD from sys.path while we load stuff.\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.read_csv(\n",
    "    DATA_PATH / \"train.csv\", \n",
    "    index_col=\"tripid\"\n",
    ")\n",
    "\n",
    "\n",
    "\n",
    "train_df['label'] = train_df['label'].map({\"correct\":1, \"incorrect\":0})\n",
    "train_df_cols = train_df.columns.values\n",
    "li = list(train_df_cols)\n",
    "features_df = train_df[li[0:-1]]\n",
    "labels_df = train_df[li[-1]]\n",
    "\n",
    "#features_df = features_df.fillna(0)\n",
    "\n",
    "features_df = formatdata(features_df)\n",
    "\n",
    "numeric_cols = features_df.columns[features_df.dtypes != object].values\n",
    "categorical_features = features_df.columns[features_df.dtypes == object].values\n",
    "\n",
    "\n",
    "\n",
    "def removeColumns(npArry, columnNames):\n",
    "    li = list(npArry)\n",
    "    for i in columnNames:\n",
    "        li.remove(i)\n",
    "    return np.array(li)\n",
    "\n",
    "\n",
    "def powerAll(data_frame, exceptList,power):\n",
    "    allColumns = data_frame.columns.values\n",
    "    for i in allColumns:\n",
    "        if(i not in exceptList ):\n",
    "            data_frame[i] = data_frame[i]**power\n",
    " \n",
    "\n",
    "#nothing\n",
    "exceptList = []\n",
    "power = 1\n",
    "\n",
    "numeric_cols = removeColumns(numeric_cols,['drop_time', 'pickup_time','drop_lat','drop_lon','pick_lat','pick_lon'\n",
    "                                             ])  \n",
    "\n",
    "powerAll(features_df, exceptList,power)\n",
    "#exceptList = features_df.columns[features_df.dtypes == \"object\"].values\n",
    "polyList = ['timediff', 'additional_fare', 'meter_waiting_fare','distance']\n",
    "numeric_preprocessing_steps = Pipeline([\n",
    "    ('standard_scaler', StandardScaler()),\n",
    "    ('simple_imputer', SimpleImputer(strategy='median'))\n",
    "])\n",
    "categorical_transformer = Pipeline(steps=[\n",
    "    ('imputer', SimpleImputer(strategy='constant', fill_value='missing')),\n",
    "    ('onehot', OneHotEncoder(handle_unknown='ignore'))])\n",
    "\n",
    "\n",
    "# create the preprocessor stage of final pipeline\n",
    "# each entry in the transformer list is a tuple of\n",
    "# (name you choose, sklearn transformer, list of columns)\n",
    "preprocessor = ColumnTransformer(\n",
    "    transformers = [\n",
    "        (\"numeric\", numeric_preprocessing_steps, numeric_cols),\n",
    "        (\"cat\", categorical_transformer, categorical_features)\n",
    "    ],\n",
    "    remainder = \"drop\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "from sklearn.ensemble import VotingClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.experimental import enable_hist_gradient_boosting\n",
    "from sklearn.ensemble import HistGradientBoostingClassifier\n",
    "from lightgbm import LGBMClassifier\n",
    "from sklearn.model_selection import KFold\n",
    "\n",
    "#estimators = MultiOutputClassifier(\n",
    " #   estimator=MLPClassifier(solver='lbfgs', alpha=.55, hidden_layer_sizes=(4,2 ), random_state=1, max_iter=10000000)\n",
    "#)\n",
    "#estimators = MultiOutputClassifier(\n",
    " #   estimator=SVC(gamma='auto', probability=True),\n",
    "#)\n",
    "#estimators =SVC(gamma='auto')\n",
    "#estimatorsY=LogisticRegression(penalty=\"l2\", C=20,solver='lbfgs',max_iter=1000000)\n",
    "solver = 'lbfgs'\n",
    "alpha = 0.075\n",
    "hidden_layer_sizes=(7,2 )\n",
    "random_state=2\n",
    "max_iter= 10000000\n",
    "activation='tanh'\n",
    "estimators=MLPClassifier(solver=solver, alpha=alpha, hidden_layer_sizes=hidden_layer_sizes, random_state=random_state,\n",
    "                         max_iter=max_iter, activation=activation)\n",
    "\n",
    "\n",
    "#estimatorsX2 = GradientBoostingClassifier(n_estimators=20, learning_rate=0.5, max_features=2, max_depth=1, random_state=0)\n",
    "min_child_weight=3\n",
    "max_depth=7\n",
    "gamma=1.5\n",
    "colsample_bytree=0.75\n",
    "subsample=0.75\n",
    "n_estimators=150\n",
    "threads = 4\n",
    "#estimators = XGBClassifier(min_child_weight=min_child_weight,max_depth=max_depth, \n",
    " #                          gamma=gamma,colsample_bytree=colsample_bytree,subsample=subsample,\n",
    "  #                         n_estimators=n_estimators, n_jobs=threads\n",
    "   #                       )\n",
    "#params = { 'axgb__n_estimators': [20, 200],'cmlp__alpha':[0,1.0],'blr__C': [1.0, 100.0]}\n",
    "#estimatorsX3 = AdaBoostClassifier(n_estimators=100, random_state=0, learning_rate = 0.05, \n",
    "#                                algorithm='SAMME.R',base_estimator=estimatorsX)\n",
    "\n",
    "#estimators = RandomForestClassifier(n_estimators=50, criterion='entropy',max_depth=10,min_samples_split=4,\n",
    " #                                  min_samples_leaf=4,max_features='log2',max_leaf_nodes=100,bootstrap=True,\n",
    "  #                                  oob_score=True,class_weight='balanced_subsample')\n",
    "#estimatorNB = GaussianNB()\n",
    "\n",
    "#estimatorsY = LGBMClassifier(boosting_type='gbdt', num_leaves=40, max_depth=6,learning_rate=0.2,\n",
    " #                          n_estimators=1000,random_state=42)\n",
    "\n",
    "#estimators = KNeighborsClassifier(n_neighbors=5, weights='distance',algorithm='brute',p=2 ,leaf_size=10)\n",
    "#estimators = BaggingClassifier(base_estimator=estimatorsY,n_estimators=10,max_samples=1.0,\n",
    " #                            max_features=1.0, random_state=1,bootstrap=False)\n",
    "#estimators = VotingClassifier( [('xgb', estimatorsX), ('mlp', estimatorsY)], weights=[1,1])\n",
    "#estimators = GridSearchCV(estimator=vestimators, param_grid=params, cv=5)\n",
    "#estimators = HistGradientBoostingClassifier(max_iter=500,min_samples_leaf=2,max_depth=10, learning_rate=0.08)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9762502413593357\n"
     ]
    }
   ],
   "source": [
    "X_train, X_eval, y_train, y_eval = train_test_split(\n",
    "    features_df,\n",
    "    labels_df,\n",
    "    test_size=0.33,\n",
    "    shuffle=True,\n",
    "    stratify=labels_df,\n",
    "    random_state=6\n",
    ")\n",
    "full_pipeline = Pipeline([\n",
    "    (\"preprocessor\", preprocessor),\n",
    "    (\"estimators\", estimators),\n",
    "])\n",
    "\n",
    "full_pipeline.fit(X_train, y_train)\n",
    "preds = full_pipeline.predict(X_eval)\n",
    "y_preds = pd.DataFrame(\n",
    "    preds,\n",
    "    index = y_eval.index\n",
    ")\n",
    "f1 = f1_score(y_eval, y_preds)\n",
    "print(f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8094\n",
      "0     482\n",
      "Name: prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "test_features_df = pd.read_csv(DATA_PATH / \"test.csv\", \n",
    "                               index_col=\"tripid\")\n",
    "\n",
    "test_features_df = formatdata(test_features_df)\n",
    "powerAll(test_features_df, exceptList,power)\n",
    "test_probas = full_pipeline.predict(test_features_df)\n",
    "test_preds = pd.DataFrame(\n",
    "    {'prediction':test_probas},\n",
    "    index = test_features_df.index\n",
    ")\n",
    "print(test_preds.prediction.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8094\n",
      "0     482\n",
      "Name: prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "print(test_preds.prediction.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9793847129717729 2\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-91-800bc36144a2>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n\u001b[0;32m     18\u001b[0m             \u001b[1;33m(\u001b[0m\u001b[1;34m\"estimators\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mestimators\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     19\u001b[0m         ])\n\u001b[1;32m---> 20\u001b[1;33m         \u001b[0mfull_pipeline1\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my_train\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     21\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     22\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\pipeline.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y, **fit_params)\u001b[0m\n\u001b[0;32m    352\u001b[0m                                  self._log_message(len(self.steps) - 1)):\n\u001b[0;32m    353\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_final_estimator\u001b[0m \u001b[1;33m!=\u001b[0m \u001b[1;34m'passthrough'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 354\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_final_estimator\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mXt\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0my\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mfit_params\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    355\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    356\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py\u001b[0m in \u001b[0;36mfit\u001b[1;34m(self, X, y)\u001b[0m\n\u001b[0;32m    993\u001b[0m         \"\"\"\n\u001b[0;32m    994\u001b[0m         return self._fit(X, y, incremental=(self.warm_start and\n\u001b[1;32m--> 995\u001b[1;33m                                             hasattr(self, \"classes_\")))\n\u001b[0m\u001b[0;32m    996\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    997\u001b[0m     \u001b[1;33m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_fit\u001b[1;34m(self, X, y, incremental)\u001b[0m\n\u001b[0;32m    373\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msolver\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'lbfgs'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    374\u001b[0m             self._fit_lbfgs(X, y, activations, deltas, coef_grads,\n\u001b[1;32m--> 375\u001b[1;33m                             intercept_grads, layer_units)\n\u001b[0m\u001b[0;32m    376\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    377\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_fit_lbfgs\u001b[1;34m(self, X, y, activations, deltas, coef_grads, intercept_grads, layer_units)\u001b[0m\n\u001b[0;32m    467\u001b[0m                     \u001b[1;34m\"gtol\"\u001b[0m\u001b[1;33m:\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtol\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    468\u001b[0m                 },\n\u001b[1;32m--> 469\u001b[1;33m                 args=(X, y, activations, deltas, coef_grads, intercept_grads))\n\u001b[0m\u001b[0;32m    470\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mn_iter_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_check_optimize_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m\"lbfgs\"\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mopt_res\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmax_iter\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    471\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mloss_\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mopt_res\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfun\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\scipy\\optimize\\_minimize.py\u001b[0m in \u001b[0;36mminimize\u001b[1;34m(fun, x0, args, method, jac, hess, hessp, bounds, constraints, tol, callback, options)\u001b[0m\n\u001b[0;32m    608\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mmeth\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'l-bfgs-b'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    609\u001b[0m         return _minimize_lbfgsb(fun, x0, args, jac, bounds,\n\u001b[1;32m--> 610\u001b[1;33m                                 callback=callback, **options)\n\u001b[0m\u001b[0;32m    611\u001b[0m     \u001b[1;32melif\u001b[0m \u001b[0mmeth\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;34m'tnc'\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    612\u001b[0m         return _minimize_tnc(fun, x0, args, jac, bounds, callback=callback,\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\scipy\\optimize\\lbfgsb.py\u001b[0m in \u001b[0;36m_minimize_lbfgsb\u001b[1;34m(fun, x0, args, jac, bounds, disp, maxcor, ftol, gtol, eps, maxfun, maxiter, iprint, callback, maxls, **unknown_options)\u001b[0m\n\u001b[0;32m    343\u001b[0m             \u001b[1;31m# until the completion of the current minimization iteration.\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    344\u001b[0m             \u001b[1;31m# Overwrite f and g:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 345\u001b[1;33m             \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunc_and_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    346\u001b[0m         \u001b[1;32melif\u001b[0m \u001b[0mtask_str\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstartswith\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34mb'NEW_X'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    347\u001b[0m             \u001b[1;31m# new iteration\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\scipy\\optimize\\lbfgsb.py\u001b[0m in \u001b[0;36mfunc_and_grad\u001b[1;34m(x)\u001b[0m\n\u001b[0;32m    293\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    294\u001b[0m         \u001b[1;32mdef\u001b[0m \u001b[0mfunc_and_grad\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 295\u001b[1;33m             \u001b[0mf\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    296\u001b[0m             \u001b[0mg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mjac\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    297\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mg\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\scipy\\optimize\\optimize.py\u001b[0m in \u001b[0;36mfunction_wrapper\u001b[1;34m(*wrapper_args)\u001b[0m\n\u001b[0;32m    325\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0mwrapper_args\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    326\u001b[0m         \u001b[0mncalls\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 327\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mfunction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mwrapper_args\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    328\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    329\u001b[0m     \u001b[1;32mreturn\u001b[0m \u001b[0mncalls\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunction_wrapper\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\scipy\\optimize\\optimize.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, x, *args)\u001b[0m\n\u001b[0;32m     63\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     64\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mnumpy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0masarray\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mcopy\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 65\u001b[1;33m         \u001b[0mfg\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfun\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mx\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     66\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mjac\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfg\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     67\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mfg\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_loss_grad_lbfgs\u001b[1;34m(self, packed_coef_inter, X, y, activations, deltas, coef_grads, intercept_grads)\u001b[0m\n\u001b[0;32m    176\u001b[0m         loss, coef_grads, intercept_grads = self._backprop(\n\u001b[0;32m    177\u001b[0m             X, y, activations, deltas, coef_grads, intercept_grads)\n\u001b[1;32m--> 178\u001b[1;33m         \u001b[0mgrad\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0m_pack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcoef_grads\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mintercept_grads\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    179\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0mloss\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mgrad\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    180\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\sklearn\\neural_network\\_multilayer_perceptron.py\u001b[0m in \u001b[0;36m_pack\u001b[1;34m(coefs_, intercepts_)\u001b[0m\n\u001b[0;32m     36\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0m_pack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mcoefs_\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mintercepts_\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     37\u001b[0m     \u001b[1;34m\"\"\"Pack the parameters into a single vector.\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 38\u001b[1;33m     \u001b[1;32mreturn\u001b[0m \u001b[0mnp\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mhstack\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m[\u001b[0m\u001b[0ml\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mravel\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0ml\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mcoefs_\u001b[0m \u001b[1;33m+\u001b[0m \u001b[0mintercepts_\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m     39\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     40\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mhstack\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;32m~\\anaconda3\\lib\\site-packages\\numpy\\core\\shape_base.py\u001b[0m in \u001b[0;36mhstack\u001b[1;34m(tup)\u001b[0m\n\u001b[0;32m    341\u001b[0m     \u001b[1;31m# As a special case, dimension 0 of 1-dimensional arrays is \"horizontal\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    342\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0marrs\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0marrs\u001b[0m\u001b[1;33m[\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mndim\u001b[0m \u001b[1;33m==\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 343\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m0\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    344\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    345\u001b[0m         \u001b[1;32mreturn\u001b[0m \u001b[0m_nx\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mconcatenate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0marrs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;36m1\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32m<__array_function__ internals>\u001b[0m in \u001b[0;36mconcatenate\u001b[1;34m(*args, **kwargs)\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "bestModel = ''\n",
    "bestscore = 0\n",
    "scores = []\n",
    "random_s = 0\n",
    "l_state = [4,5,7,8,9,20,80,78,0,1,2,6,3,10,42]    \n",
    "for i in range(2,3):\n",
    "    \n",
    "    kf = KFold(n_splits=10, random_state=i, shuffle=True)\n",
    "\n",
    "    kf.get_n_splits(features_df)\n",
    "\n",
    "    for train_index, test_index in kf.split(features_df):\n",
    "        X_train, X_eval, y_train, y_eval = features_df.iloc[train_index], features_df.iloc[test_index],labels_df.iloc[train_index], labels_df.iloc[test_index]\n",
    "        \n",
    "       \n",
    "        full_pipeline1 = Pipeline([\n",
    "            (\"preprocessor\", preprocessor),\n",
    "            (\"estimators\", estimators),\n",
    "        ])\n",
    "        full_pipeline1.fit(X_train, y_train)\n",
    "        \n",
    "        \n",
    "        preds = full_pipeline1.predict(X_eval)\n",
    "        y_preds = pd.DataFrame(\n",
    "            preds,\n",
    "            index = y_eval.index\n",
    "        )\n",
    "        f1 = f1_score(y_eval, y_preds)\n",
    "        if(f1>bestscore):\n",
    "            bestscore = f1\n",
    "            bestModel = full_pipeline1\n",
    "        print(f1,i)\n",
    "\n",
    "        \n",
    "\n",
    "    print(i)\n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8108\n",
      "0     468\n",
      "Name: prediction, dtype: int64\n"
     ]
    }
   ],
   "source": [
    "#bestModel.fit(features_df, labels_df)\n",
    "test_features_df = pd.read_csv(DATA_PATH / \"test.csv\", \n",
    "                               index_col=\"tripid\")\n",
    "#test_features_df = test_features_df.fillna(0)\n",
    "test_features_df = formatdata(test_features_df)\n",
    "powerAll(test_features_df, exceptList,power)\n",
    "#exceptList = features_df.columns[features_df.dtypes == \"object\"].values\n",
    "#test_features_df = formatdata(test_features_df)\n",
    "\n",
    "\n",
    "\n",
    "test_probas = bestModel.predict(test_features_df)\n",
    "test_preds1 = pd.DataFrame(\n",
    "    {'prediction':test_probas},\n",
    "    index = test_features_df.index\n",
    ")\n",
    "\n",
    "print(test_preds1.prediction.value_counts())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1    8108\n",
      "0     468\n",
      "Name: prediction, dtype: int64\n",
      "False\n",
      "0.9778200253485424\n"
     ]
    }
   ],
   "source": [
    "print(test_preds1.prediction.value_counts())\n",
    "print (test_preds1.equals(test_preds))\n",
    "print(bestscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import KFold\n",
    "from sklearn.metrics import f1_score\n",
    "kf = KFold(n_splits=8, random_state=1, shuffle=True)\n",
    "\n",
    "kf.get_n_splits(features_df)\n",
    "\n",
    "tempbs=0\n",
    "tempbm=''\n",
    "\n",
    "for train_index, test_index in kf.split(features_df):\n",
    "\n",
    "    X_train, X_eval, y_train, y_eval = features_df.iloc[train_index], features_df.iloc[test_index],labels_df.iloc[train_index], labels_df.iloc[test_index]\n",
    "    full_pipeline = Pipeline([\n",
    "        (\"preprocessor\", preprocessor),\n",
    "        (\"estimators\", estimators),\n",
    "    ])\n",
    "    full_pipeline.fit(X_train, y_train)\n",
    "    preds = full_pipeline.predict(X_eval)\n",
    "    y_preds = pd.DataFrame(\n",
    "        preds,\n",
    "        index = y_eval.index\n",
    "    )\n",
    "\n",
    "    f1 = f1_score(y_eval, y_preds)\n",
    "    \n",
    "    if(f1 > tempbs):\n",
    "        tempbs = f1\n",
    "        tempbm = full_pipeline\n",
    "        \n",
    "    print(f1)\n",
    "    \n",
    "print('done')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tempbm.fit(features_df, labels_df)\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_features_df = pd.read_csv(DATA_PATH / \"test.csv\", \n",
    "                               index_col=\"tripid\")\n",
    "test_features_df = test_features_df.fillna(0)\n",
    "test_features_df = formatdata(test_features_df)\n",
    "powerAll(test_features_df, exceptList,power)\n",
    "#exceptList = features_df.columns[features_df.dtypes == \"object\"].values\n",
    "#test_features_df = formatdata(test_features_df)\n",
    "\n",
    "test_probas = tempbm.predict(test_features_df)\n",
    "test_preds = pd.DataFrame(\n",
    "    {'prediction':test_probas},\n",
    "    index = test_features_df.index\n",
    ")\n",
    "\n",
    "\n",
    "test_preds.to_csv(DATA_PATH / 'my_submission.csv', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(tempbs)\n",
    "selectedList = []\n",
    "for i in range(len(scores)):\n",
    "    if scores[i]>= 0.98:\n",
    "        selectedList.append([i, scores[i]])\n",
    "\n",
    "print(selectedList)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(bestscore)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "confusion_matrix(y_eval, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#0.9782128197031891\n",
    "from sklearn.metrics import recall_score\n",
    "recall= recall_score(y_eval, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import precision_score\n",
    "precission = precision_score(y_eval, y_preds)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import f1_score\n",
    "f1 = f1_score(y_eval, y_preds)\n",
    "print(recall)\n",
    "print(precission)\n",
    "print(f1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if(type(estimators) ==  XGBClassifier):\n",
    "    f = open(DATA_PATH / 'xgboostStatus.txt','a')\n",
    "    f.write(\"recall: \"+ str(recall)+\"\\t\"+\"precission: \"+ str(precission)+\"\\tf1: \"+str(f1)+\"\\n\")\n",
    "    f.write(\"min_child_weight: \"+str(min_child_weight)+\"\\tmax_depth:\"+ str(max_depth)+\"\\tgamma: \"+str(gamma)+\n",
    "            \"\\tcolsample_bytree: \"+str(colsample_bytree)+\"\\tsubsample: \"+str(subsample) +\"\\tn_estimators\"+ str(n_estimators)+\n",
    "            \"\\power: \"+ str(power)+\"\\n\\n\")\n",
    "    f.close()\n",
    "\n",
    "if(type(estimators) == MLPClassifier):\n",
    "    f = open(DATA_PATH / 'mlpStatus.txt','a')\n",
    "    f.write(\"recall: \"+ str(recall)+\"\\t\"+\"precission: \"+ str(precission)+\"\\tf1: \"+str(f1)+\"\\n\")\n",
    "    f.write(\"solver: \"+str(solver)+\"\\talpha: \"+ str(alpha)+\"\\thidden_layer_sizes: \"+str(hidden_layer_sizes)+\n",
    "            \"\\trandom_state: \"+str(random_state)+\"\\tmax_iter: \"+str(max_iter) +\"\\tlearning_rate_init: \"+ str(\"learning_rate_init\")+\n",
    "            \"\\tlearning_rate: \"+ str(\"learning_rate\")+\"\\tactivation: \"+ str(activation)+\"\\power: \"+ str(power)+\"\\n\\n\")\n",
    "    f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%time \n",
    "\n",
    "tempbm.fit(features_df, labels_df)\n",
    "\n",
    "None"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_features_df = pd.read_csv(DATA_PATH / \"test.csv\", \n",
    "                               index_col=\"tripid\")\n",
    "#test_features_df = test_features_df.fillna(0)\n",
    "test_features_df = formatdata(test_features_df)\n",
    "powerAll(test_features_df, exceptList,power)\n",
    "#exceptList = features_df.columns[features_df.dtypes == \"object\"].values\n",
    "#test_features_df = formatdata(test_features_df)\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_probas = tempbm.predict(test_features_df)\n",
    "\n",
    "test_probas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds = pd.DataFrame(\n",
    "    {'prediction':test_probas},\n",
    "    index = test_features_df.index\n",
    ")\n",
    "\n",
    "print(test_preds['prediction'].max())\n",
    "print(test_preds['prediction'].min())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_preds.to_csv(DATA_PATH / 'my_submission.csv', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make sure we have the rows in the same order\n",
    "np.testing.assert_array_equal(test_features_df.index.values, \n",
    "                              submission_df.index.values)\n",
    "\n",
    "# Save predictions to submission data frame\n",
    "submission_df[\"h1n1_vaccine\"] = test_preds[1]\n",
    "\n",
    "submission_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission_df.to_csv(DATA_PATH / 'my_submission.csv', index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "to = '11/01/2019  00:20:00'\n",
    "to += '.099'\n",
    "froms = '11/01/2019  00:34:00'+'.099'\n",
    "dateObjectto = datetime.strptime(to, '%m/%d/%Y %H:%M:%S.%f')\n",
    "dateObjectfrom = datetime.strptime(froms, '%m/%d/%Y %H:%M:%S.%f')\n",
    "print(datetime.timestamp(dateObjectfrom) - datetime.timestamp(dateObjectto))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = pd.read_csv(\n",
    "    DATA_PATH / \"train.csv\", \n",
    "    index_col=\"tripid\"\n",
    ")\n",
    "train_df['label'] = train_df['label'].map({\"correct\":1, \"incorrect\":0})\n",
    "train_df_cols = train_df.columns.values\n",
    "li = list(train_df_cols)\n",
    "features_df = train_df[li[0:-1]]\n",
    "labels_df = train_df[li[-1]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "features_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(features_df.dtypes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "features_df[features_df.dtypes[(features_df.dtypes==\"float64\")]\n",
    "                        .index.values].hist(figsize=[11,11],\n",
    "                                            range=[features_df['drop_lat'].min(), features_df['drop_lat'].max()])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "cols_to_norm = ['duration','meter_waiting',\n",
    "                'meter_waiting_fare','meter_waiting_till_pickup',\n",
    "                'pick_lat','pick_lon','drop_lat','drop_lon','fare']\n",
    "n_test = features_df[cols_to_norm]\n",
    "\n",
    "\n",
    "x = n_test.values\n",
    "\n",
    "min_max_scaler = preprocessing.MinMaxScaler()\n",
    "x_scaled = min_max_scaler.fit_transform(x)\n",
    "\n",
    "n_test = pd.DataFrame(x_scaled, columns=cols_to_norm,index=features_df.index)\n",
    "\n",
    "l_test = features_df.drop(cols_to_norm, axis=1)\n",
    "train = pd.concat([n_test, l_test], axis=1)\n",
    "train.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "def plot():\n",
    "    plt.figure(figsize=(8,6))\n",
    "\n",
    "    plt.scatter(features_df['meter_waiting_fare'], labels_df,\n",
    "            color='green', label='input scale', alpha=0.5)\n",
    "\n",
    "    \n",
    "    plt.xlabel('Alcohol')\n",
    "    plt.ylabel('Malic Acid')\n",
    "    plt.legend(loc='upper left')\n",
    "    plt.grid()\n",
    "\n",
    "    plt.tight_layout()\n",
    "\n",
    "    \n",
    "plot()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import seaborn as sns\n",
    "train_df = pd.read_csv(\n",
    "    DATA_PATH / \"train.csv\", \n",
    "    index_col=\"tripid\"\n",
    ")\n",
    "train_df['label'] = train_df['label'].map({\"correct\":1, \"incorrect\":0})\n",
    "train_df_cols = train_df.columns.values\n",
    "li = list(train_df_cols)\n",
    "features_df = train_df[li[0:-1]]\n",
    "labels_df = train_df[li[-1]]\n",
    "features_df = formatdata(features_df)\n",
    "features_df = features_df.join(labels_df)\n",
    "\n",
    "labels_df.fillna(0)\n",
    "features_df[features_df==np.inf]=np.nan\n",
    "features_df.fillna(0, inplace=True)\n",
    "numericOnle = ['duration','meter_waiting','meter_waiting_fare','meter_waiting_till_pickup','fare','timediff','distance','label']\n",
    "features_set = features_df[numericOnle]\n",
    "\n",
    "#get correlations of each features in dataset\n",
    "corrmat = features_set.corr()\n",
    "top_corr_features = corrmat.index\n",
    "plt.figure(figsize=(8,8))\n",
    "#plot heat map\n",
    "g=sns.heatmap(features_set[top_corr_features].corr(),annot=True,cmap=\"RdYlGn\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}