Rakuten_modele(3).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "Rakuten_modele(3).ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/freezer2019/Rakuten-Paris-Product-Classification/blob/main/Rakuten_modele(3).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KFpSDQdgAzOM",
        "outputId": "8bdd4a6d-c79c-4565-c80f-e13696f64896"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install torch torchvision\n",
        "!pip install sentencepiece"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (4.1.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: tokenizers==0.9.4 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.9.4)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.8)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.4)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.6/dist-packages (1.7.0+cu101)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.6/dist-packages (0.8.1+cu101)\n",
            "Requirement already satisfied: dataclasses in /usr/local/lib/python3.6/dist-packages (from torch) (0.8)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.6/dist-packages (from torch) (3.7.4.3)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.6/dist-packages (from torch) (0.16.0)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from torch) (1.19.4)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.6/dist-packages (from torchvision) (7.0.0)\n",
            "Requirement already satisfied: sentencepiece in /usr/local/lib/python3.6/dist-packages (0.1.94)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WEJnN-1PBrV4",
        "outputId": "18a07d6e-c0a3-4f63-9760-8e8be7c8a71b"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np \n",
        "import nltk\n",
        "from nltk import WordPunctTokenizer\n",
        "nltk.download('stopwords')\n",
        "from nltk.corpus import stopwords\n",
        "import seaborn as sns\n",
        "import os\n",
        "import torch\n",
        "import torch.optim as optim\n",
        "from torch.optim import lr_scheduler\n",
        "from transformers import CamembertModel, CamembertTokenizer, CamembertConfig\n",
        "import matplotlib.pyplot as plt\n",
        "import warnings\n",
        "warnings.filterwarnings('ignore')\n",
        "from sklearn.pipeline import FeatureUnion, Pipeline, make_pipeline\n",
        "from sklearn.base import BaseEstimator, TransformerMixin\n",
        "import torch.nn as nn \n",
        "from skimage import io, transform\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torchvision import models, transforms, utils\n",
        "import random\n",
        "import cv2\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score\n",
        "from sklearn.model_selection import cross_val_score\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "from sklearn.preprocessing import StandardScaler\n",
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "import torch.nn.functional as F"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ousHq398pZhh"
      },
      "source": [
        "import sklearn.ensemble as ske"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ACso4dJ9pixG"
      },
      "source": [
        "import sklearn.discriminant_analysis as d_a"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_MXqkr0_pHAJ"
      },
      "source": [
        "#LinearDiscriminantAnalysis(solver='svd', shrinkage=None, priors=None, n_components=None, store_covariance=False, tol=0.0001, covariance_estimator=None)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_enIKKECi5C"
      },
      "source": [
        "import time\n",
        "import copy"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VuvHY0S9fRnz"
      },
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\n",
        "from sklearn.linear_model import SGDClassifier"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dgomrPKmBp8m"
      },
      "source": [
        "from google.colab import drive,files\n",
        "drive.mount(\"/content/drive\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "egoFX4t9D2vP"
      },
      "source": [
        "!unzip /content/drive/MyDrive/DS2021/Datasets/archive\\(8\\).zip "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bQ4BEz1ED3wd"
      },
      "source": [
        "x_train = pd.read_csv(\"/content/X_train_update.csv\")\n",
        "y_train = pd.read_csv(\"/content/Y_train_CVw08PX.csv\")\n",
        "x_test = pd.read_csv(\"/content/X_test_update.csv\")"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NPhsmO0ZEH1S"
      },
      "source": [
        "x_train.rename(columns = {\"Unnamed: 0\": \"Id\"}, inplace = True)\n",
        "y_train.rename(columns = {\"Unnamed: 0\": \"Id\"}, inplace = True)\n",
        "x_test.rename(columns = {\"Unnamed: 0\": \"Id\"}, inplace = True)\n",
        "x_train[\"root_dir\"]=x_train[\"Id\"].apply(lambda x: \"/content/images/images/image_train\")\n",
        "x_test[\"root_dir\"]=x_train[\"Id\"].apply(lambda x: \"/content/images/images/image_test\")\n"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 293
        },
        "id": "RmQif5-bkO1R",
        "outputId": "4f99aef7-bc99-4d7c-cba1-9d220a83b40b"
      },
      "source": [
        "x_train.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Id</th>\n",
              "      <th>designation</th>\n",
              "      <th>description</th>\n",
              "      <th>productid</th>\n",
              "      <th>imageid</th>\n",
              "      <th>root_dir</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Olivia: Personalisiertes Notizbuch / 150 Seite...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>3804725264</td>\n",
              "      <td>1263597046</td>\n",
              "      <td>/content/images/images/image_train</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>Journal Des Arts (Le) N° 133 Du 28/09/2001 - L...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>436067568</td>\n",
              "      <td>1008141237</td>\n",
              "      <td>/content/images/images/image_train</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>Grand Stylet Ergonomique Bleu Gamepad Nintendo...</td>\n",
              "      <td>PILOT STYLE Touch Pen de marque Speedlink est ...</td>\n",
              "      <td>201115110</td>\n",
              "      <td>938777978</td>\n",
              "      <td>/content/images/images/image_train</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>Peluche Donald - Europe - Disneyland 2000 (Mar...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>50418756</td>\n",
              "      <td>457047496</td>\n",
              "      <td>/content/images/images/image_train</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>La Guerre Des Tuques</td>\n",
              "      <td>Luc a des id&amp;eacute;es de grandeur. Il veut or...</td>\n",
              "      <td>278535884</td>\n",
              "      <td>1077757786</td>\n",
              "      <td>/content/images/images/image_train</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   Id  ...                            root_dir\n",
              "0   0  ...  /content/images/images/image_train\n",
              "1   1  ...  /content/images/images/image_train\n",
              "2   2  ...  /content/images/images/image_train\n",
              "3   3  ...  /content/images/images/image_train\n",
              "4   4  ...  /content/images/images/image_train\n",
              "\n",
              "[5 rows x 6 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N0qhpi6-lLQO"
      },
      "source": [
        "Helpers function, Classes\n",
        "\n",
        "```\n",
        "# This is formatted as code\n",
        "```\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "91VfE49sFs0H"
      },
      "source": [
        "class simple_Text_cleaner(BaseEstimator, TransformerMixin):\n",
        "  # add another additional parameter, just for fun, while we are at it\n",
        "    def __init__(self, stopwords,columns,tokenizer=WordPunctTokenizer()): \n",
        "        self.columns = columns\n",
        "        self.stopwords = stopwords\n",
        "        self.tokenizer=tokenizer\n",
        "    def rm_stopwords(self,tokens):\n",
        "        tokens=self.tokenizer.tokenize(tokens)\n",
        "        return [ tk for tk in tokens if  tk not in self.stopwords ]\n",
        "    def text_clean_up(self,s=\"\"):\n",
        "        import re\n",
        "        user_pattern       = '@[^\\s]+'\n",
        "        s=re.sub(user_pattern, \"\", s)\n",
        "        remove = '\"#$%&()*+/:;<=>@[\\\\]^_`{|}~”“'\n",
        "        pattern = r\"[{}]\".format(remove)\n",
        "        s=re.sub(pattern,' ', s) \n",
        "        sequencePattern   = r\"(.)\\1\\1+\"\n",
        "        seqReplacePattern = r\"\\1\\1\"\n",
        "        s = re.sub(sequencePattern, seqReplacePattern, s)\n",
        "        s = re.sub(\"<[^>]*>\",' ', s)\n",
        "        s = re.sub(\"[\\r\\n]+\",' ', s)\n",
        "        s = re.sub(\"http\\S+\",' ', s)\n",
        "        s = re.sub(\"\\$[^>]*\\$\",' ', s)\n",
        "        s = re.sub(\"\\d+\",' ', s)\n",
        "        s = re.sub(\"\\s\\s+\",' ', s)\n",
        "        s.strip()\n",
        "        return s\n",
        "    def fit(self, X, y=None):\n",
        "        return self\n",
        "    def transform(self, X, y=None):\n",
        "        X_ = X.copy()\n",
        "        X_ [\"image_path\"]=X.apply(lambda x: generate_path(x[\"root_dir\"], x[\"imageid\"], x[\"productid\"]),axis=1)\n",
        "        for i in self.columns:\n",
        "            X_[i] = X_[i].apply(lambda x: \" \".join(self.rm_stopwords(self.text_clean_up(x))))\n",
        "        return X_.drop([\"root_dir\", \"imageid\", \"productid\"], axis=1)"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "soYHl8tOBNgG"
      },
      "source": [
        "def encode_reviews(tokenizer, model, device, cpu, reviews, max_length,batch):\n",
        "    token_ids = torch.tensor([])\n",
        "    input_ids=torch.tensor([],dtype=torch.long)\n",
        "    attention_mask=torch.tensor([],dtype=torch.long)\n",
        "    h = 0\n",
        "    for i, review in enumerate(reviews):\n",
        "        encoded = encoded_text = tokenizer.encode_plus(\n",
        "                        review,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = max_length,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "        input_ids=torch.cat((input_ids,encoded_text['input_ids']),dim=0)\n",
        "        attention_mask=torch.cat((attention_mask,encoded_text['attention_mask']),dim=0)\n",
        "        h+=1\n",
        "        if h == batch:\n",
        "            hidden = model(input_ids.to(device),attention_mask=attention_mask.to(device))[2][-2]\n",
        "            token_ids=torch.cat((token_ids,torch.mean(hidden, dim=1).to(cpu)),dim=0)\n",
        "            h=0\n",
        "            input_ids=torch.tensor([],dtype=torch.long)\n",
        "            attention_mask=torch.tensor([],dtype=torch.long)\n",
        "    if len(input_ids) != 0:\n",
        "        hidden = model(input_ids.to(device),attention_mask=attention_mask.to(device))[2][-2]\n",
        "        token_ids=torch.cat((token_ids,torch.mean(hidden, dim=1).to(cpu)),dim=0)\n",
        "        print(token_ids.size())\n",
        "    token_ids=token_ids.numpy()\n",
        "    print(token_ids.shape)\n",
        "    return token_ids\n",
        "def encode_text(tokenizer, model, device, cpu, text, max_length,batch):\n",
        "    encoded = encoded_text = tokenizer.encode_plus(\n",
        "                    text,                      # Sentence to encode.\n",
        "                    add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                    max_length = max_length,           # Pad & truncate all sentences.\n",
        "                    pad_to_max_length = True,\n",
        "                    return_attention_mask = True,   # Construct attn. masks.\n",
        "                    return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                )\n",
        "    input_ids=encoded_text['input_ids']\n",
        "    attention_mask=encoded_text['attention_mask']\n",
        "    hidden = model(input_ids.to(device),attention_mask=attention_mask.to(device))[2]\n",
        "    x=hidden[-2]\n",
        "    #x=torch.cat((hidden[-4], hidden[-3], hidden[-2], hidden[-1]), dim = 2)\n",
        "    token_ids=torch.mean(x, dim=1).to(cpu).squeeze(0)\n",
        "    token_ids=token_ids.numpy()\n",
        "    return token_ids"
      ],
      "execution_count": 43,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gdT0SN64GIxp"
      },
      "source": [
        "class CamembertPreprocessor(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self, max_seq_length,column,bs=128):\n",
        "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "        self.cpu=torch.device(\"cpu\")\n",
        "        print(self.device)\n",
        "        self.tokenizer = CamembertTokenizer.from_pretrained(\"camembert-base\")\n",
        "        self.config = CamembertConfig.from_pretrained(\"camembert-base\", output_hidden_states=True)\n",
        "        self.camembert =CamembertModel.from_pretrained(\"camembert-base\", config=self.config).to(self.device)\n",
        "        for p in self.camembert.parameters():\n",
        "                p.requires_grad_(False)\n",
        "        self.max_seq_length = max_seq_length\n",
        "        self.column = column\n",
        "        self.camembert.eval()\n",
        "        self.batch=bs\n",
        "    def fit(self, X=None):\n",
        "        return self \n",
        "    \n",
        "    def transform(self, X, y=None):\n",
        "        # 1. Tokenize\n",
        "        since = time.time()\n",
        "        #X_encoded = encode_reviews(self.tokenizer,self.camembert,self.device,self.cpu, X[self.column].values, self.max_seq_length,self.batch)\n",
        "        if y is None:\n",
        "            seq_length=64\n",
        "        else:\n",
        "            seq_length=self.max_seq_length\n",
        "        X_encoded = X[self.column].apply(lambda x: pd.Series(encode_text(self.tokenizer,self.camembert,self.device,self.cpu, x, seq_length,self.batch)))\n",
        "        time_elapsed = time.time() - since\n",
        "        print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
        "        X_encoded=np.array(X_encoded)\n",
        "        print(len(X_encoded))\n",
        "        print(len(X_encoded[0]))\n",
        "        print(X_encoded.shape)\n",
        "        return X_encoded  \n",
        "    \n",
        "    def fit_transform(self, X, y=None):        \n",
        "        return self.transform(X, y)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b4DmnNcp6KFt"
      },
      "source": [
        "\n",
        "torch.cat((torch.tensor([]),torch.tensor([1,2])),dim=0)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h1WTwE-WGI7r"
      },
      "source": [
        "class Second_to_last_SentenceEmbedding(nn.Module):\n",
        "    def __init__(self,to_tune):\n",
        "        #super(, self).__init__()\n",
        "        super().__init__()\n",
        "        self.config = CamembertConfig.from_pretrained(\"camembert-base\", output_hidden_states=True)\n",
        "        self.camembert = CamembertModel.from_pretrained(\"camembert-base\", config=self.config)\n",
        "        dense_layer1 = nn.Sequential(nn.Linear(768,100),nn.ReLU(),nn.Dropout(p=0.2))\n",
        "        dense_layer2 = nn.Sequential(nn.Linear(100,54),nn.ReLU(),nn.Dropout(p=0.2))\n",
        "        fc2=nn.Linear(54,27)\n",
        "        self.dense=nn.Sequential(dense_layer1, dense_layer2, fc2)\n",
        "        if not to_tune:\n",
        "            for p in self.camembert.parameters():\n",
        "                p.requires_grad_(False)\n",
        "    def forward(self, input,attention_mask=None):\n",
        "        hidden = self.camembert(input,attention_mask=attention_mask)[2]\n",
        "        token_vecs = hidden[-2]\n",
        "        x = torch.mean(token_vecs,dim=1)\n",
        "        return self.dense(x)"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EcZS8g4NGJEJ"
      },
      "source": [
        "class Columns_Selector(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self,column):\n",
        "        self.column = column\n",
        "    def fit(self, X = None):\n",
        "        return self \n",
        "    def transform(self, X, y=None):\n",
        "        return X[self.column]     \n",
        "    def fit_transform(self, X, y=None):        \n",
        "        return self.transform(X, y)"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6X0rqUT3Q8dz"
      },
      "source": [
        "def generate_path(path,image_id,product_id):\n",
        "    return os.path.join(path,f\"image_{image_id}_product_{product_id}.jpg\")\n",
        "class Image_path(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self,column):\n",
        "        self.column = column\n",
        "    def fit(self, X = None):\n",
        "        return self \n",
        "    def transform(self, X, y=None):\n",
        "        X[\"image_path\"]=X.apply(lambda x: generate_path(x[\"root_dir\"], x[\"imageid\"], x[\"productid\"]),axis=1)\n",
        "        X.drop([\"root_dir\", \"imageid\", \"productid\"], axis=1)\n",
        "        return X.drop([\"root_dir\", \"imageid\", \"productid\"], axis=1)\n",
        "    def fit_transform(self, X, y=None):        \n",
        "        return self.transform(X, y)"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pal4xfSiGJLT"
      },
      "source": [
        "class To_dense(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self):\n",
        "        pass\n",
        "    def fit(self, X = None):\n",
        "        return self \n",
        "    def transform(self, X, y=None):\n",
        "        X = X.toarray()\n",
        "        print(X.shape)\n",
        "        return X \n",
        "    def fit_transform(self, X, y=None):        \n",
        "        return self.transform(X, y)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oS7yq_DWHRIQ"
      },
      "source": [
        "class MultimodalDataset (Dataset):\n",
        "    def __init__(self,tokenizer, input_csv_file, root_dir,max_length=60 , transform=None,Y_csv_file=None):\n",
        "        if type(input_csv_file) == str:\n",
        "            self.input_file = pd.read_csv(input_csv_file)\n",
        "        else:\n",
        "            self.input_file = input_csv_file\n",
        "        self.input_file.rename(columns={\"Unnamed: 0\": \"Id\"}, inplace=True)\n",
        "        self.root_dir = root_dir\n",
        "        self.transform =transform\n",
        "        self.max_length=max_length\n",
        "        self.input_file[\"image_path\"]=(self.input_file[\"imageid\"].apply(lambda x: str(int(x)))+\"_product_\"+self.input_file[\"productid\"].apply(lambda x: str(int(x)))+\".jpg\").apply(lambda x: \"image_\"+x)\n",
        "        if Y_csv_file is not None:\n",
        "            if type(Y_csv_file) == str:\n",
        "                self.output = pd.read_csv(Y_csv_file)\n",
        "                \n",
        "            else:\n",
        "                self.output = Y_csv_file\n",
        "            self.output.rename(columns={\"Unnamed: 0\": \"Id\"}, inplace=True)\n",
        "            self.classes = list(set(self.output[\"prdtypecode\"].values))\n",
        "        else:\n",
        "            self.ouptut = None\n",
        "    def __len__(self):\n",
        "        return len(self.input_file)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        if torch.is_tensor(idx):\n",
        "            idx = idx.tolist()\n",
        "        try:\n",
        "            img_name = os.path.join(self.root_dir,\n",
        "                                    self.input_file[\"image_path\"].iloc[idx])\n",
        "        except:\n",
        "            print(idx)\n",
        "\n",
        "        image = io.imread(img_name)\n",
        "        text = self.input_file[\"designation\"].iloc[idx]\n",
        "        encoded_text = tokenizer.encode_plus(\n",
        "                        text,                      # Sentence to encode.\n",
        "                        add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                        max_length = 64,           # Pad & truncate all sentences.\n",
        "                        pad_to_max_length = True,\n",
        "                        return_attention_mask = True,   # Construct attn. masks.\n",
        "                        return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                   )\n",
        "        sample = {'image': image, 'input_ids':encoded_text['input_ids'].squeeze(0) ,'attention_mask':encoded_text['attention_mask'].squeeze(0)}\n",
        "        sample['Id']=self.input_file[\"Id\"].iloc[idx]\n",
        "        if self.transform:\n",
        "            sample['image'] = self.transform(sample['image'])\n",
        "        if self.output is None:\n",
        "            return sample\n",
        "        sample[\"label\"]=self.classes.index(self.output[\"prdtypecode\"].loc[self.input_file[\"Id\"].iloc[idx]])\n",
        "        return sample"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M49aG8Z0bnyE"
      },
      "source": [
        "def encoder_multimodal(tokenizer, model, device, cpu, x, max_length,transform):\n",
        "    encoded = encoded_text = tokenizer.encode_plus(\n",
        "                    x[\"designation\"],                      # Sentence to encode.\n",
        "                    add_special_tokens = True, # Add '[CLS]' and '[SEP]'\n",
        "                    max_length = max_length,           # Pad & truncate all sentences.\n",
        "                    pad_to_max_length = True,\n",
        "                    return_attention_mask = True,   # Construct attn. masks.\n",
        "                    return_tensors = 'pt',     # Return pytorch tensors.\n",
        "                )\n",
        "    input_ids = encoded_text['input_ids']\n",
        "    attention_mask = encoded_text['attention_mask']\n",
        "    image = transform(io.imread(x[\"image_path\"]))\n",
        "    hidden = torch.cat(model(input = input_ids.to(device),\n",
        "                             image = image.to(device),\n",
        "                             attention_mask = attention_mask.to(device)),dim=1)\n",
        "    \n",
        "    token_ids = hidden.numpy()\n",
        "    return token_ids"
      ],
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OgApZ_05HZrV"
      },
      "source": [
        "class Microscope:\n",
        "    \"\"\"\n",
        "    Cutting out the edges around the center circle of the image\n",
        "    Imitating a picture, taken through the microscope\n",
        "\n",
        "    Args:\n",
        "        p (float): probability of applying an augmentation\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, p: float = 0.5):\n",
        "        self.p = p\n",
        "\n",
        "    def __call__(self, img):\n",
        "        \"\"\"\n",
        "        Args:\n",
        "            img (PIL Image): Image to apply transformation to.\n",
        "\n",
        "        Returns:NumPyNumPy\n",
        "            PIL Image: Image with transformation.\n",
        "        \"\"\"\n",
        "        if random.random() < self.p:\n",
        "            circle = cv2.circle((np.ones(img.shape) * 255).astype(np.uint8), # image placeholder\n",
        "                        (img.shape[0]//2, img.shape[1]//2), # center point of circle\n",
        "                        random.randint(img.shape[0]//2 - 3, img.shape[0]//2 + 15), # radius\n",
        "                        (0, 0, 0), # color\n",
        "                        -1)\n",
        "\n",
        "            mask = circle - 255\n",
        "            img = np.multiply(img, mask)\n",
        "        \n",
        "        return img\n",
        "\n",
        "    def __repr__(self):\n",
        "        return f'{self.__class__.__name__}(p={self.p})'"
      ],
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WLZuV956HxZj"
      },
      "source": [
        "class Feature_Extraction(nn.Module):\n",
        "    # deuxieme dimension de l'output de chaque tenseurs a la sortie d'un layer de resnet\n",
        "    resnet_caracteristics=[64,64,64,64,256,512,1024,2048,2048]\n",
        "    pooling_target=[(4,8),(4,8),(4,8),(4,8),(2,4),(2,2),(1,2),(1,1),(1,1)]\n",
        "    embedding_strategy={\"Second_to_last_average\":768,\"Start_token_embedding\":768,\"last_four_embedding_average\":3072}\n",
        "    def __init__(self,resnet_layers=5,to_tune=False,sentence_embedding=\"Second_to_last_average\"):\n",
        "        #super(, self).__init__()\n",
        "        super().__init__()\n",
        "        self.config = CamembertConfig.from_pretrained(\"camembert-base\", output_hidden_states=True)\n",
        "        self.camembert =CamembertModel.from_pretrained(\"camembert-base\", config=self.config)\n",
        "        self.Resnet = models.resnet50(pretrained=True)\n",
        "        self.Resnet = nn.Sequential(*list(self.Resnet.children())[:resnet_layers])\n",
        "        self.pooling = nn.AdaptiveAvgPool2d(self.pooling_target[resnet_layers-1])\n",
        "        self.flat = nn.Flatten()\n",
        "        self.strategy = sentence_embedding\n",
        "        if sentence_embedding not in self.embedding_strategy:\n",
        "            self.strategy = \"Second_to_last_average\"    \n",
        "        if self.strategy == \"Start_token_embedding\":\n",
        "            to_tune = True\n",
        "        if to_tune == False:\n",
        "            for p in self.camembert.parameters():\n",
        "                p.requires_grad_(False)\n",
        "        for p in self.Resnet.parameters():\n",
        "                p.requires_grad_(False)\n",
        "        self.output_size = [2048, self.embedding_strategy[self.strategy]]\n",
        "    def sentence_embedding(self,hiddens):\n",
        "        # we get the output of the second to last hidden layers and average it over all token\n",
        "        if self.strategy == \"Second_to_last_average\":\n",
        "            return torch.mean(hiddens[-2],dim=1) \n",
        "        # we use the first token embedding  from the ouptut last hidden layers \n",
        "        # fine_tune should be true \n",
        "        elif self.strategy == \"Start_token_embedding\":\n",
        "            return hiddens[-1].permute(1,0,2)[0]\n",
        "        # we use the last four hidden layer average and concatenate them\n",
        "        elif self.strategy == \"last_four_embedding_average\":\n",
        "            x=torch.cat((hiddens[-4], hiddens[-3], hiddens[-2], hiddens[-1]), dim = 2)\n",
        "            return torch.mean(x, dim=1)\n",
        "    def forward(self, input, image, attention_mask=None):\n",
        "        hiddens = self.camembert(input,attention_mask = attention_mask)[2]\n",
        "        embeddings = self.sentence_embedding(hiddens)\n",
        "        x = self.Resnet(image)\n",
        "        h = self.pooling(x)\n",
        "        return embeddings, self.flat(h)"
      ],
      "execution_count": 24,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sDvOmbM6sROm"
      },
      "source": [
        "class Camembert_resnet_Preprocessor(BaseEstimator, TransformerMixin):\n",
        "    def __init__(self, max_seq_length,resnet_layers=5,to_tune=False,sentence_embedding=\"Second_to_last_average\"):\n",
        "        self.device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "        self.cpu=torch.device(\"cpu\")\n",
        "        print(self.device)\n",
        "        self.tokenizer = CamembertTokenizer.from_pretrained(\"camembert-base\")\n",
        "        self.feature_extractor = Feature_Extraction(resnet_layers = resnet_layers,\n",
        "                                                    to_tune = to_tune,\n",
        "                                                    sentence_embedding = sentence_embedding).to(self.device)\n",
        "        self.max_seq_length = max_seq_length\n",
        "        self.feature_extractor.eval()\n",
        "        self.data_transform=transforms.Compose([transforms.ToPILImage(),\n",
        "                                    transforms.Resize(256),\n",
        "                                    transforms.ToTensor(),\n",
        "                                    transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "                                ])\n",
        "    def fit(self, X=None):\n",
        "        return self \n",
        "    \n",
        "    def transform(self, X, y=None):\n",
        "        # 1. Tokenize\n",
        "        since = time.time()\n",
        "        #X_encoded = encode_reviews(self.tokenizer,self.camembert,self.device,self.cpu, X[self.column].values, self.max_seq_length,self.batch)\n",
        "        if y is None:\n",
        "            seq_length=64\n",
        "        else:\n",
        "            seq_length=self.max_seq_length\n",
        "        X_encoded = X.apply(lambda x: pd.Series(encoder_multimodal(self.tokenizer,self.feature_extractor,self.device,self.cpu, x, seq_length,self.data_transform)))\n",
        "        time_elapsed = time.time() - since\n",
        "        print('Training complete in {:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
        "        X_encoded=np.array(X_encoded)\n",
        "        print(len(X_encoded))\n",
        "        print(len(X_encoded[0]))\n",
        "        print(X_encoded.shape)\n",
        "        return X_encoded  \n",
        "    \n",
        "    def fit_transform(self, X, y=None):        \n",
        "        return self.transform(X, y)"
      ],
      "execution_count": 25,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cgcK01jlH6bC"
      },
      "source": [
        "class Multimodal_Dense_model(nn.Module):\n",
        "    def __init__(self,dropout=0.2,resnet_layers=-1,to_tune=False,sentence_embedding=\"Second_to_last_average\"):\n",
        "        #super(, self).__init__()\n",
        "        super().__init__()\n",
        "        self.feature_extractor = Feature_Extraction(resnet_layers=resnet_layers,to_tune=to_tune,sentence_embedding=sentence_embedding)\n",
        "        self.input_size  = self.feature_extractor.output_size\n",
        "        self.dense_layer = nn.Sequential(OrderedDict([\n",
        "          ('dense1', nn.Linear(in_features=sum(self.input_size),out_features=1024)),\n",
        "          ('relu1', nn.ReLU()),\n",
        "          ('dropout1', nn.Dropout(p=0.1)),\n",
        "          ('dense2', nn.Linear(in_features=1024,out_features=256)),\n",
        "          ('relu2', nn.ReLU()),\n",
        "          ('dropout2', nn.Dropout(p=0.1)),\n",
        "          ('dense3', nn.Linear(in_features=256,out_features=64)),\n",
        "          ('relu3', nn.ReLU()),\n",
        "          ('dropout3', nn.Dropout(p=0.1)),\n",
        "          ('output', nn.Linear(64,27))\n",
        "          ]))\n",
        "    def forward(self, input,image,attention_mask=None):\n",
        "        self.feature_extractor.eval()\n",
        "        text,image = self.feature_extractor(input=input,image=image,attention_mask=attention_mask)\n",
        "        return self.dense_layer(torch.cat((text,image),dim=1))"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sUaOu1RCIYk6"
      },
      "source": [
        "data_transforms = {\n",
        "    'train': transforms.Compose([\n",
        "        Microscope(p=0.5),\n",
        "        transforms.ToPILImage(),\n",
        "        transforms.RandomResizedCrop(size=384, scale=(0.8, 1.0)),\n",
        "        transforms.RandomHorizontalFlip(),\n",
        "        transforms.RandomVerticalFlip(),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "    'val': transforms.Compose([\n",
        "        transforms.Resize(256),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "    'test': transforms.Compose([\n",
        "        transforms.ToPILImage(),\n",
        "        transforms.Resize(256),\n",
        "        transforms.ToTensor(),\n",
        "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
        "    ]),\n",
        "}"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AJ9fwFN5IBfC"
      },
      "source": [
        "tokenizer = CamembertTokenizer.from_pretrained(\"camembert-base\")\n",
        "training_set = MultimodalDataset(tokenizer=tokenizer, input_csv_file=\"/content/X_train_update.csv\", root_dir=\"/content/images/images/image_train\",max_length=60 , transform=data_transforms[\"train\"],Y_csv_file=\"/content/Y_train_CVw08PX.csv\")\n",
        "def make_weights_for_balanced_classes(images, nclasses):                                                                            \n",
        "    count = list(images.output[\"prdtypecode\"].value_counts().loc[images.classes])                                    \n",
        "    N = float(sum(count)) \n",
        "    weight_per_class=[0]*nclasses                                                  \n",
        "    for i in range(nclasses):                                                   \n",
        "        weight_per_class[i] = N/float(count[i])                                 \n",
        "    weight = [0] * len(images)                                              \n",
        "    for idx in range(int(N)):    \n",
        "        h=images.classes.index(images.output[\"prdtypecode\"].iloc[idx])                                      \n",
        "        weight[idx] = weight_per_class[h]                                  \n",
        "    return weight\n",
        "weights = make_weights_for_balanced_classes(training_set, len(training_set.classes))                                                                \n",
        "weights = torch.DoubleTensor(weights)                                       \n",
        "sampler = torch.utils.data.sampler.WeightedRandomSampler(weights, len(weights))                     \n",
        "                                                                                                                                                                        \n",
        "train_dl = torch.utils.data.DataLoader(training_set, batch_size=16,                              \n",
        "                                                             sampler = sampler, num_workers=4)"
      ],
      "execution_count": 28,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vIv1iRG8Ku6l"
      },
      "source": [
        "Baseline model\n",
        "Tf-idfVectorizer + Logistics Regression (Texte)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1dA388wCFyG3"
      },
      "source": [
        "X_train,X_Val,Y_train,Y_Val=train_test_split(x_train, y_train, test_size = 0.1, shuffle = True , random_state = 102)"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nZ8cYRrEZCSf"
      },
      "source": [
        "target = Y_train[\"prdtypecode\"]"
      ],
      "execution_count": 30,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eNGe90VVlsqd"
      },
      "source": [
        "?LogisticRegression"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "fIkxcZhC8IjR"
      },
      "source": [
        "stop_words = stopwords.words('french')\n",
        "stop_words.extend(stopwords.words('english'))"
      ],
      "execution_count": 31,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NvZL6ieUGH0i"
      },
      "source": [
        "Base_model = Pipeline(steps = [('cleaner', simple_Text_cleaner(stopwords =stop_words, columns = [\"designation\"])),\n",
        "                               ('column_selector', Columns_Selector(column = \"designation\")),\n",
        "                               ('Vectorization', TfidfVectorizer(stop_words = stop_words, max_features = 5000)),\n",
        "                               ('to_dense', To_dense()),\n",
        "                               ('scaler', StandardScaler()),\n",
        "                               ('model', LogisticRegression(multi_class='multinomial',class_weight= \"balanced\"))\n",
        "                           ])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3fZ7aiHPUib2",
        "outputId": "151cc96c-dbf7-4b05-9b7a-2e0ca00656ea"
      },
      "source": [
        "Base_model.fit(X_train,target)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(76424, 5000)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(memory=None,\n",
              "         steps=[('cleaner',\n",
              "                 simple_Text_cleaner(columns=['designation'],\n",
              "                                     stopwords=['au', 'aux', 'avec', 'ce',\n",
              "                                                'ces', 'dans', 'de', 'des',\n",
              "                                                'du', 'elle', 'en', 'et', 'eux',\n",
              "                                                'il', 'ils', 'je', 'la', 'le',\n",
              "                                                'les', 'leur', 'lui', 'ma',\n",
              "                                                'mais', 'me', 'même', 'mes',\n",
              "                                                'moi', 'mon', 'ne', 'nos', ...],\n",
              "                                     tokenizer=WordPunctTokenizer(pattern='\\\\w+|[^\\\\w\\\\s]+', gaps=False, discard_empt...\n",
              "                ('to_dense', To_dense()),\n",
              "                ('scaler',\n",
              "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
              "                ('model',\n",
              "                 LogisticRegression(C=1.0, class_weight='balanced', dual=False,\n",
              "                                    fit_intercept=True, intercept_scaling=1,\n",
              "                                    l1_ratio=None, max_iter=100,\n",
              "                                    multi_class='multinomial', n_jobs=None,\n",
              "                                    penalty='l2', random_state=None,\n",
              "                                    solver='lbfgs', tol=0.0001, verbose=0,\n",
              "                                    warm_start=False))],\n",
              "         verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5b5yoJsiWm4r",
        "outputId": "c6ba68d4-8304-4461-a192-424640d6c4ec"
      },
      "source": [
        "pred = Base_model.predict(X_Val)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(8492, 5000)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7jLUb-R9UweH",
        "outputId": "a0a759a8-24c0-4148-e514-a4bbe5891e5d"
      },
      "source": [
        "print(classification_report(Y_Val[\"prdtypecode\"],pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "          10       0.26      0.23      0.24       301\n",
            "          40       0.59      0.55      0.57       259\n",
            "          50       0.72      0.71      0.72       186\n",
            "          60       0.83      0.80      0.82        97\n",
            "        1140       0.69      0.63      0.66       254\n",
            "        1160       0.86      0.86      0.86       362\n",
            "        1180       0.43      0.49      0.46        67\n",
            "        1280       0.58      0.54      0.56       491\n",
            "        1281       0.36      0.44      0.40       193\n",
            "        1300       0.87      0.85      0.86       549\n",
            "        1301       0.90      0.88      0.89        90\n",
            "        1302       0.73      0.73      0.73       258\n",
            "        1320       0.67      0.65      0.66       325\n",
            "        1560       0.74      0.75      0.74       488\n",
            "        1920       0.85      0.85      0.85       419\n",
            "        1940       0.66      0.67      0.67        89\n",
            "        2060       0.73      0.73      0.73       485\n",
            "        2220       0.67      0.59      0.63        83\n",
            "        2280       0.68      0.58      0.63       507\n",
            "        2403       0.72      0.70      0.71       454\n",
            "        2462       0.60      0.72      0.65       123\n",
            "        2522       0.85      0.87      0.86       470\n",
            "        2582       0.62      0.61      0.62       287\n",
            "        2583       0.95      0.94      0.94      1000\n",
            "        2585       0.67      0.70      0.68       260\n",
            "        2705       0.38      0.60      0.46       287\n",
            "        2905       1.00      0.96      0.98       108\n",
            "\n",
            "    accuracy                           0.72      8492\n",
            "   macro avg       0.69      0.69      0.69      8492\n",
            "weighted avg       0.72      0.72      0.72      8492\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c9eobnxLcd6r"
      },
      "source": [
        "[link text](https:// [link text](https://))Bert extraction De feature "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FiWBXb1UcacO",
        "outputId": "0b96fdeb-11fe-484a-e2b0-3080bc23946a"
      },
      "source": [
        "Simple_text_model = Pipeline(steps = [('cleaner', simple_Text_cleaner(stopwords =stop_words, columns = [\"designation\"])),\n",
        "                               ('Vectorization', CamembertPreprocessor(max_seq_length =40 , column = \"designation\")),\n",
        "                               ('scaler', StandardScaler()),\n",
        "                               ('model', LogisticRegression(multi_class='multinomial',class_weight= \"balanced\"))\n",
        "                           ])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5cebwAKptA2R",
        "outputId": "0349ed78-41f7-4ef3-c9dc-bd9d513d7e0d"
      },
      "source": [
        "Simple_text_model.fit(X_train,target)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Training complete in 17m 19s\n",
            "76424\n",
            "3072\n",
            "(76424, 3072)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(memory=None,\n",
              "         steps=[('cleaner',\n",
              "                 simple_Text_cleaner(columns=['designation'],\n",
              "                                     stopwords=['au', 'aux', 'avec', 'ce',\n",
              "                                                'ces', 'dans', 'de', 'des',\n",
              "                                                'du', 'elle', 'en', 'et', 'eux',\n",
              "                                                'il', 'ils', 'je', 'la', 'le',\n",
              "                                                'les', 'leur', 'lui', 'ma',\n",
              "                                                'mais', 'me', 'même', 'mes',\n",
              "                                                'moi', 'mon', 'ne', 'nos', ...],\n",
              "                                     tokenizer=WordPunctTokenizer(pattern='\\\\w+|[^\\\\w\\\\s]+', gaps=False, discard_empt...\n",
              "                                       max_seq_length=40)),\n",
              "                ('scaler',\n",
              "                 StandardScaler(copy=True, with_mean=True, with_std=True)),\n",
              "                ('model',\n",
              "                 LogisticRegression(C=1.0, class_weight='balanced', dual=False,\n",
              "                                    fit_intercept=True, intercept_scaling=1,\n",
              "                                    l1_ratio=None, max_iter=100,\n",
              "                                    multi_class='multinomial', n_jobs=None,\n",
              "                                    penalty='l2', random_state=None,\n",
              "                                    solver='lbfgs', tol=0.0001, verbose=0,\n",
              "                                    warm_start=False))],\n",
              "         verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iszkVEhlJJ_L",
        "outputId": "58509d7f-b5c8-4385-d2cf-e393539ea85a"
      },
      "source": [
        "pred = Simple_text_model.predict(X_Val)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training complete in 1m 56s\n",
            "8492\n",
            "3072\n",
            "(8492, 3072)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eZCYVA05MHdh",
        "outputId": "3587f5f0-13d6-42bd-8af8-a19b02739b45"
      },
      "source": [
        "print(classification_report(Y_Val[\"prdtypecode\"],pred))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "          10       0.46      0.48      0.47       301\n",
            "          40       0.52      0.60      0.56       259\n",
            "          50       0.70      0.76      0.73       186\n",
            "          60       0.86      0.72      0.79        97\n",
            "        1140       0.57      0.62      0.60       254\n",
            "        1160       0.87      0.86      0.86       362\n",
            "        1180       0.31      0.57      0.40        67\n",
            "        1280       0.66      0.41      0.51       491\n",
            "        1281       0.38      0.50      0.43       193\n",
            "        1300       0.90      0.86      0.88       549\n",
            "        1301       0.84      0.81      0.82        90\n",
            "        1302       0.60      0.74      0.66       258\n",
            "        1320       0.61      0.64      0.63       325\n",
            "        1560       0.75      0.71      0.73       488\n",
            "        1920       0.88      0.85      0.86       419\n",
            "        1940       0.75      0.75      0.75        89\n",
            "        2060       0.77      0.69      0.73       485\n",
            "        2220       0.52      0.81      0.63        83\n",
            "        2280       0.88      0.72      0.79       507\n",
            "        2403       0.72      0.71      0.71       454\n",
            "        2462       0.51      0.67      0.58       123\n",
            "        2522       0.87      0.84      0.85       470\n",
            "        2582       0.67      0.60      0.63       287\n",
            "        2583       0.94      0.95      0.94      1000\n",
            "        2585       0.86      0.80      0.82       260\n",
            "        2705       0.54      0.72      0.62       287\n",
            "        2905       1.00      0.98      0.99       108\n",
            "\n",
            "    accuracy                           0.74      8492\n",
            "   macro avg       0.70      0.72      0.70      8492\n",
            "weighted avg       0.75      0.74      0.74      8492\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "qW8k3WT0XmRH",
        "outputId": "4f26beca-b50e-4da1-b7bc-0f9f961e525e"
      },
      "source": [
        "print(classification_report(Y_train[\"prdtypecode\"],Simple_text_model.predict(X_train)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Training complete in 17m 16s\n",
            "76424\n",
            "3072\n",
            "(76424, 3072)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "          10       0.66      0.72      0.69      2815\n",
            "          40       0.70      0.82      0.75      2249\n",
            "          50       0.81      0.89      0.85      1495\n",
            "          60       0.97      0.88      0.92       735\n",
            "        1140       0.73      0.81      0.77      2417\n",
            "        1160       0.91      0.90      0.90      3591\n",
            "        1180       0.49      0.89      0.63       697\n",
            "        1280       0.77      0.48      0.59      4379\n",
            "        1281       0.55      0.67      0.60      1877\n",
            "        1300       0.92      0.90      0.91      4496\n",
            "        1301       0.96      0.97      0.97       717\n",
            "        1302       0.67      0.88      0.76      2233\n",
            "        1320       0.71      0.80      0.76      2916\n",
            "        1560       0.83      0.79      0.81      4585\n",
            "        1920       0.90      0.88      0.89      3884\n",
            "        1940       0.87      0.98      0.92       714\n",
            "        2060       0.84      0.70      0.76      4508\n",
            "        2220       0.66      0.96      0.78       741\n",
            "        2280       0.95      0.80      0.87      4253\n",
            "        2403       0.84      0.79      0.82      4320\n",
            "        2462       0.78      0.92      0.85      1298\n",
            "        2522       0.92      0.86      0.89      4519\n",
            "        2582       0.79      0.72      0.75      2302\n",
            "        2583       0.98      0.96      0.97      9209\n",
            "        2585       0.91      0.92      0.91      2236\n",
            "        2705       0.68      0.89      0.77      2474\n",
            "        2905       1.00      0.99      1.00       764\n",
            "\n",
            "    accuracy                           0.83     76424\n",
            "   macro avg       0.81      0.84      0.82     76424\n",
            "weighted avg       0.84      0.83      0.83     76424\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TLbpmLnrXtcQ",
        "outputId": "58211384-9986-44ac-af34-1b968056477c"
      },
      "source": [
        "print(classification_report(Y_train[\"prdtypecode\"],Base_model.predict(X_train)))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "(76424, 5000)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "          10       0.74      0.64      0.69      2815\n",
            "          40       0.91      0.88      0.90      2249\n",
            "          50       0.98      0.98      0.98      1495\n",
            "          60       0.98      0.99      0.99       735\n",
            "        1140       0.93      0.94      0.94      2417\n",
            "        1160       0.98      0.96      0.97      3591\n",
            "        1180       0.80      0.95      0.87       697\n",
            "        1280       0.95      0.87      0.91      4379\n",
            "        1281       0.81      0.89      0.85      1877\n",
            "        1300       0.98      0.97      0.98      4496\n",
            "        1301       0.99      1.00      0.99       717\n",
            "        1302       0.96      0.97      0.97      2233\n",
            "        1320       0.96      0.96      0.96      2916\n",
            "        1560       0.98      0.96      0.97      4585\n",
            "        1920       0.97      0.99      0.98      3884\n",
            "        1940       0.89      0.97      0.93       714\n",
            "        2060       0.98      0.96      0.97      4508\n",
            "        2220       0.97      0.99      0.98       741\n",
            "        2280       0.88      0.82      0.85      4253\n",
            "        2403       0.92      0.81      0.86      4320\n",
            "        2462       0.94      0.96      0.95      1298\n",
            "        2522       0.99      0.98      0.99      4519\n",
            "        2582       0.96      0.98      0.97      2302\n",
            "        2583       1.00      0.98      0.99      9209\n",
            "        2585       0.96      0.97      0.96      2236\n",
            "        2705       0.53      0.84      0.65      2474\n",
            "        2905       1.00      1.00      1.00       764\n",
            "\n",
            "    accuracy                           0.93     76424\n",
            "   macro avg       0.92      0.93      0.93     76424\n",
            "weighted avg       0.94      0.93      0.93     76424\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "aQek3nNoco5K",
        "outputId": "cd2bd82d-740a-407a-ab6d-34b5dd239c33"
      },
      "source": [
        "Simple_text_sgdmodel = Pipeline(steps = [('cleaner', simple_Text_cleaner(stopwords =stop_words, columns = [\"designation\"])),\n",
        "                               ('Vectorization', CamembertPreprocessor(max_seq_length = 32, column = \"designation\")),\n",
        "                               ('scaler', StandardScaler()),\n",
        "                               ('model', SGDClassifier(class_weight= \"balanced\"))\n",
        "                           ])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "l5s-yBL2M7MU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1a13a4ce-d48c-42d5-b8d3-64c26d10680a"
      },
      "source": [
        "Simple_text_sgdmodel.fit(X_train,target)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Training complete in 17m 11s\n",
            "76424\n",
            "3072\n",
            "(76424, 3072)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "Pipeline(memory=None,\n",
              "         steps=[('cleaner',\n",
              "                 simple_Text_cleaner(columns=['designation'],\n",
              "                                     stopwords=['au', 'aux', 'avec', 'ce',\n",
              "                                                'ces', 'dans', 'de', 'des',\n",
              "                                                'du', 'elle', 'en', 'et', 'eux',\n",
              "                                                'il', 'ils', 'je', 'la', 'le',\n",
              "                                                'les', 'leur', 'lui', 'ma',\n",
              "                                                'mais', 'me', 'même', 'mes',\n",
              "                                                'moi', 'mon', 'ne', 'nos', ...],\n",
              "                                     tokenizer=WordPunctTokenizer(pattern='\\\\w+|[^\\\\w\\\\s]+', gaps=False, discard_empt...\n",
              "                 SGDClassifier(alpha=0.0001, average=False,\n",
              "                               class_weight='balanced', early_stopping=False,\n",
              "                               epsilon=0.1, eta0=0.0, fit_intercept=True,\n",
              "                               l1_ratio=0.15, learning_rate='optimal',\n",
              "                               loss='hinge', max_iter=1000, n_iter_no_change=5,\n",
              "                               n_jobs=None, penalty='l2', power_t=0.5,\n",
              "                               random_state=None, shuffle=True, tol=0.001,\n",
              "                               validation_fraction=0.1, verbose=0,\n",
              "                               warm_start=False))],\n",
              "         verbose=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 43
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T93zgLoagwsg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f8193d1-91aa-4eca-b0ad-7f457455aa8c"
      },
      "source": [
        "print(\"============Bert============\")\n",
        "print(classification_report(Y_Val[\"prdtypecode\"],Simple_text_sgdmodel.predict(X_Val),))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "============Bert============\n",
            "Training complete in 1m 52s\n",
            "8492\n",
            "3072\n",
            "(8492, 3072)\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "          10       0.53      0.39      0.45       301\n",
            "          40       0.47      0.47      0.47       259\n",
            "          50       0.48      0.74      0.58       186\n",
            "          60       0.91      0.63      0.74        97\n",
            "        1140       0.58      0.56      0.57       254\n",
            "        1160       0.83      0.76      0.79       362\n",
            "        1180       0.17      0.63      0.27        67\n",
            "        1280       0.53      0.40      0.45       491\n",
            "        1281       0.28      0.41      0.34       193\n",
            "        1300       0.94      0.61      0.74       549\n",
            "        1301       0.77      0.80      0.79        90\n",
            "        1302       0.53      0.65      0.58       258\n",
            "        1320       0.42      0.74      0.54       325\n",
            "        1560       0.87      0.53      0.66       488\n",
            "        1920       0.83      0.81      0.82       419\n",
            "        1940       0.62      0.79      0.69        89\n",
            "        2060       0.77      0.62      0.68       485\n",
            "        2220       0.64      0.63      0.63        83\n",
            "        2280       0.87      0.76      0.81       507\n",
            "        2403       0.77      0.69      0.72       454\n",
            "        2462       0.61      0.62      0.61       123\n",
            "        2522       0.82      0.75      0.78       470\n",
            "        2582       0.65      0.50      0.57       287\n",
            "        2583       0.84      0.95      0.89      1000\n",
            "        2585       0.84      0.67      0.75       260\n",
            "        2705       0.48      0.84      0.61       287\n",
            "        2905       0.99      0.90      0.94       108\n",
            "\n",
            "    accuracy                           0.68      8492\n",
            "   macro avg       0.67      0.66      0.65      8492\n",
            "weighted avg       0.72      0.68      0.68      8492\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aOl1aavEmwoT"
      },
      "source": [
        "from sklearn.naive_bayes import MultinomialNB"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VxzpJjCelyLD"
      },
      "source": [
        "\n",
        "\n",
        "> Indented block\n",
        "Random Forest Model \n",
        "using Bert "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7ccVFHHBlxl8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "58c6ba0a-ef7a-424b-cfa5-68204775fe48"
      },
      "source": [
        "Simple_text_xgbmodel = Pipeline(steps = [('cleaner', simple_Text_cleaner(stopwords =stop_words, columns = [\"designation\"])),\n",
        "                               ('Vectorization', CamembertPreprocessor(max_seq_length = 40, column = \"designation\")),\n",
        "                               ('scaler', StandardScaler()),\n",
        "                               ('lda',d_a.LinearDiscriminantAnalysis(n_components=100)),\n",
        "                               ('model', RandomForestClassifier(n_estimators=100, n_jobs=-1, random_state=10, verbose=1,class_weight=\"balanced\"))\n",
        "                           ])"
      ],
      "execution_count": 44,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "cuda\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-EM5TSkunH6k",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8bd7df37-8295-4517-8604-e6d9152723f3"
      },
      "source": [
        "Simple_text_xgbmodel.fit(X_train,target)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Training complete in 15m 21s\n",
            "76424\n",
            "768\n",
            "(76424, 768)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "[Parallel(n_jobs=-1)]: Using backend ThreadingBackend with 2 concurrent workers.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jNcz8p1knJkl"
      },
      "source": [
        "print(\"============Bert============\")\n",
        "print(classification_report(Simple_text_xgbmodel.predict(X_Val),Y_Val[\"prdtypecode\"]))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z6Kcu-dlJKo8"
      },
      "source": [
        "Perceptron Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Jid6b-c6wwQs"
      },
      "source": [
        "General Train Function-"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v2BSNuHOwvME"
      },
      "source": [
        "soft_max = nn.LogSoftmax(dim=1)\n",
        "import numpy as np\n",
        "from sklearn.metrics import f1_score\n",
        "def accuracy(out, yb):\n",
        "    preds = torch.argmax(soft_max(out), dim=1) # classe prédite: tensor de taille bs. \n",
        "    score=f1_score(yb.to(cpu).numpy(),preds.to(cpu).numpy(), average=\"weighted\")\n",
        "    return score\n",
        "def train_model(model, criterion, optimizer, scheduler, num_epochs=25):\n",
        "    since = time.time()\n",
        "    best_model_wts = copy.deepcopy(model.state_dict())\n",
        "    best_acc = 0.0\n",
        "    all_loss={\"train\":[],\"val\":[]}\n",
        "    all_acc={\"train\":[],\"val\":[]}\n",
        "    all_acc_all={\"train\":[],\"val\":[]}\n",
        "    for epoch in range(num_epochs):\n",
        "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
        "        print('-' * 10)\n",
        "\n",
        "        # Each epoch has a training and validation phase\n",
        "        for phase in ['train', 'val']:\n",
        "            if phase == 'train':\n",
        "                model.train()  # Set model to training mode\n",
        "            else:\n",
        "                model.eval()   # Set model to evaluate mode\n",
        "\n",
        "            running_loss = 0.0\n",
        "            running_corrects = 0\n",
        "            all_labels=torch.tensor([],dtype=torch.float32)\n",
        "            all_ouputs=torch.tensor([],dtype=torch.float32).to(device)\n",
        "            # Iterattention_maskate over data.\n",
        "            for inputs in dataloaders[phase]:\n",
        "                input_ids = inputs[\"input_ids\"].to(device)\n",
        "                attention_mask = inputs[\"attention_mask\"].to(device)\n",
        "                all_labels=torch.cat((all_labels,inputs[\"label\"]),dim=0)\n",
        "                labels = inputs[\"label\"].to(device)\n",
        "                # zero the parameter gradients\n",
        "                optimizer.zero_grad()\n",
        "                \n",
        "                # forward\n",
        "                # track history if only in train\n",
        "                with torch.set_grad_enabled(phase == 'train'):\n",
        "                    outputs = model(input_ids, attention_mask = attention_mask)\n",
        "                    loss = criterion(outputs, labels , weight  = torch.tensor(weights,dtype = torch.float32).to(device))\n",
        "                    all_ouputs=torch.cat((all_ouputs,outputs),dim=0)\n",
        "                    # backward + optimize only if in training phase\n",
        "                    if phase == 'train':\n",
        "                        loss.backward()\n",
        "                        optimizer.step()\n",
        "\n",
        "                # statistics\n",
        "                running_loss += loss.item() * input_ids.size(0)\n",
        "                running_corrects += accuracy(outputs ,labels)* input_ids.size(0)\n",
        "            if phase == 'train':\n",
        "                scheduler.step()\n",
        "\n",
        "            epoch_loss = running_loss / len(dataset[phase])\n",
        "            epoch_acc = running_corrects / len(dataset[phase])\n",
        "            epoch_acc_all = accuracy(all_ouputs ,all_labels)\n",
        "            all_loss[phase].append(epoch_loss)\n",
        "            all_acc[phase].append(epoch_acc)\n",
        "            all_acc_all[phase].append(epoch_acc_all)\n",
        "            print('{} Loss: {:.4f} Acc: {:.4f} , Acc_all: {:.4f}'.format(\n",
        "                phase, epoch_loss, epoch_acc,epoch_acc_all))\n",
        "\n",
        "            # deep copy the model\n",
        "            if phase == 'val' and epoch_acc_all > best_acc:\n",
        "                best_acc = epoch_acc_all\n",
        "                best_model_wts = copy.deepcopy(model.state_dict())\n",
        "\n",
        "        print()\n",
        "\n",
        "    time_elapsed = time.time() - since\n",
        "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
        "        time_elapsed // 60, time_elapsed % 60))\n",
        "    print('Best val Acc: {:4f}'.format(best_acc))\n",
        "\n",
        "    # load best model weights\n",
        "    model.load_state_dict(best_model_wts)\n",
        "    return model, all_loss, all_acc,all_acc_all"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1iKTtFwMxzJN"
      },
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "cpu=torch.device(\"cpu\")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cPr67u-a6yLg"
      },
      "source": [
        "Torch Dataset et DataLoaders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nsNVE9tE63SG"
      },
      "source": [
        "dataset={}\n",
        "dataloaders={}\n",
        "dataset[\"full\"] = MultimodalDataset(tokenizer=tokenizer, input_csv_file=\"/content/X_train_update.csv\", root_dir=\"/content/images/images/image_train\",max_length=64 , transform=data_transforms[\"train\"],Y_csv_file=\"/content/Y_train_CVw08PX.csv\")                     \n",
        "dataloaders[\"full\"] = torch.utils.data.DataLoader(dataset[\"full\"], batch_size=128,shuffle=True, num_workers=6)#balanced dataset\n",
        "\n",
        "dataset[\"train\"]=MultimodalDataset(tokenizer=tokenizer, input_csv_file=X_train, root_dir=\"/content/images/images/image_train\",max_length=108 , transform=data_transforms[\"train\"],Y_csv_file=Y_train)         \n",
        "dataloaders[\"train\"] = torch.utils.data.DataLoader(dataset[\"train\"], batch_size=128,shuffle=True, num_workers=6)#balanced dataset\n",
        "\n",
        "dataset[\"val\"]=MultimodalDataset(tokenizer=tokenizer, input_csv_file=X_Val, root_dir=\"/content/images/images/image_train\",max_length=108 , transform=data_transforms[\"train\"],Y_csv_file=Y_Val)                                                           \n",
        "dataloaders[\"val\"] = torch.utils.data.DataLoader(dataset[\"val\"], batch_size=128,shuffle=True, num_workers=6)#balanced dataset"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "PTOQTk0zJRpe"
      },
      "source": [
        "**Text** Perceptron\n",
        "\n",
        "```\n",
        "# This is formatted as code\n",
        "```\n",
        "On va utiliser la classe Second_to_last_embeddings\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rvYW-zMEJQwv"
      },
      "source": [
        "Simple_text_model_untuned=Second_to_last_SentenceEmbedding(to_tune=False).to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OJQZrDbw6w-5"
      },
      "source": [
        "optimizer_untuned = optim.SGD(Simple_text_model_untuned.dense.parameters(), lr=0.005, momentum=0.9)\n",
        "\n",
        "exp_lr_scheduler_untuned = lr_scheduler.StepLR(optimizer_untuned, step_size=7, gamma=0.1)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SdwykZ6RIBJn"
      },
      "source": [
        "loss_function= F.cross_entropy"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4cttnefVqJXA"
      },
      "source": [
        "training the model without tuning the Bert"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rkZphNGz_bVN"
      },
      "source": [
        "weights=compute_class_weight(\"balanced\", classes= np.unique(y_train[\"prdtypecode\"]), y=y_train[\"prdtypecode\"])\n",
        "h=np.unique(target)\n",
        "weights_dict={}\n",
        "for i in range(len(h)):\n",
        "    weights_dict[h[i]] = weights[i]\n",
        "    "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RE3DuMYtpF2U"
      },
      "source": [
        "Simple_text_model_untuned,untuned_loss,untuned_acc,untuned_acc_all = train_model(Simple_text_model_untuned, \n",
        "                                                   loss_function, optimizer_untuned, exp_lr_scheduler_untuned, num_epochs=24)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lAyjSQbLw3_g"
      },
      "source": [
        "fig,axs=plt.subplots(1,2,figsize=(10,10))\n",
        "untuned_loss,untuned_acc\n",
        "axs[0].plot(list(range(len(untuned_loss[\"train\"]))),untuned_loss[\"train\"],label=\"Train\")\n",
        "axs[0].plot(list(range(len(untuned_loss[\"val\"]))),untuned_loss[\"val\"],label=\"Valid\")\n",
        "axs[0].set_title(\"Loss\")\n",
        "axs[1].plot(list(range(len(untuned_acc[\"train\"]))),untuned_acc[\"train\"],label=\"Train\")\n",
        "axs[1].plot(list(range(len(untuned_acc[\"val\"]))),untuned_acc[\"val\"],label=\"Valid\")\n",
        "axs[1].set_title(\"F1_Score\")\n",
        "plt.legend()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "j8BvD7A1qWcH"
      },
      "source": [
        "Tuning Bert"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sElDEHNvqcDE"
      },
      "source": [
        "Tuning Big Model "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RuvdxT9Gqq2p"
      },
      "source": [
        "from collections import OrderedDict\n",
        "model_1 = Multimodal_Dense_model(resnet_layers = 6, to_tune = True ,sentence_embedding = \"last_four_embedding_average\").to(device)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BQIvR51KtNUE"
      },
      "source": [
        "optimizer_1 = optim.SGD(model_1.parameters(), lr=0.001, momentum=0.9)\n",
        "\n",
        "exp_lr_scheduler_1 = lr_scheduler.StepLR(optimizer_1, step_size=7, gamma=0.1)\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qB2boQQJrzxz"
      },
      "source": [
        "def train_model_multi(model, criterion, optimizer, scheduler, num_epochs=25):\n",
        "    since = time.time()\n",
        "\n",
        "    best_model_wts = copy.deepcopy(model.state_dict())\n",
        "    best_acc = 0.0\n",
        "    all_loss={\"train\":[],\"val\":[]}\n",
        "    all_acc={\"train\":[],\"val\":[]}\n",
        "    for epoch in range(num_epochs):\n",
        "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
        "        print('-' * 10)\n",
        "\n",
        "        # Each epoch has a training and validation phase\n",
        "        for phase in ['train', 'val']:\n",
        "            if phase == 'train':\n",
        "                model.train()  # Set model to training mode\n",
        "            else:\n",
        "                model.eval()   # Set model to evaluate mode\n",
        "\n",
        "            running_loss = 0.0\n",
        "            running_corrects = 0\n",
        "\n",
        "            # Iterattention_maskate over data.\n",
        "            for inputs in dataloaders[phase]:\n",
        "                image = inputs[\"image\"].to(device)\n",
        "                input_ids = inputs[\"input_ids\"].to(device)\n",
        "                attention_mask = inputs[\"attention_mask\"].to(device)\n",
        "                labels = inputs[\"label\"].to(device)\n",
        "\n",
        "                # zero the parameter gradients\n",
        "                optimizer.zero_grad()\n",
        "                \n",
        "                # forward\n",
        "                # track history if only in train\n",
        "                with torch.set_grad_enabled(phase == 'train'):\n",
        "                    outputs = model(input_ids,image = image, attention_mask = attention_mask)\n",
        "                    loss = criterion(outputs, labels, weight=torch.tensor(weights,dtype = torch.float32).to(device))\n",
        "\n",
        "                    # backward + optimize only if in training phase\n",
        "                    if phase == 'train':\n",
        "                        loss.backward()\n",
        "                        optimizer.step()\n",
        "\n",
        "                # statistics\n",
        "                running_loss += loss.item() * input_ids.size(0)\n",
        "                running_corrects += accuracy(outputs ,labels)*input_ids.size(0)\n",
        "            if phase == 'train':\n",
        "                scheduler.step()\n",
        "\n",
        "            epoch_loss = running_loss / len(dataset[phase])\n",
        "            epoch_acc = running_corrects.double() /len(dataset[phase])\n",
        "            all_loss[phase].append(epoch_loss)\n",
        "            all_acc[phase].append(epoch_acc)\n",
        "            print('{} Loss: {:.4f} Acc: {:.4f}'.format(\n",
        "                phase, epoch_loss, epoch_acc))\n",
        "\n",
        "            # deep copy the model\n",
        "            if phase == 'val' and epoch_acc > best_acc:\n",
        "                best_acc = epoch_acc\n",
        "                best_model_wts = copy.deepcopy(model.state_dict())\n",
        "\n",
        "        print()\n",
        "\n",
        "    time_elapsed = time.time() - since\n",
        "    print('Training complete in {:.0f}m {:.0f}s'.format(\n",
        "        time_elapsed // 60, time_elapsed % 60))\n",
        "    print('Best val Acc: {:4f}'.format(best_acc))\n",
        "\n",
        "    # load best model weights\n",
        "    model.load_state_dict(best_model_wts)\n",
        "    return model, all_loss, all_acc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ICsWDKRmttkd"
      },
      "source": [
        "model_1,loss_1,acc_1 = train_model_multi(model_1, loss_function, optimizer_1, exp_lr_scheduler_1, num_epochs=25)"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}