Code-Example-015.1--Gradient-Descents-Vectorized.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import time\n",
    "import psutil\n",
    "import matplotlib.pyplot as plt \n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAAM8UlEQVR4nO3da4hcdx3G8edxN0VrlSoZNTZdV4MINaAtQ7daEGkVahWbF76oS70hBAW1XkAaYS3SN74Q8YIooVYrditSy1pKvBRtKYKuzqZVt66Xeluj0YyKrTeojT9fzBGSye7OmTln58xv8/3AkpkzZ+b/8GPycHJ2JscRIQBAPk9oOgAAYDQUOAAkRYEDQFIUOAAkRYEDQFLT41xs9+7dMTs7O84lASC9lZWVP0dEq3/7WAt8dnZWnU5nnEsCQHq2f7vRdk6hAEBSFDgAJEWBA0BSFDgAJEWBA0BSAwvc9i22T9hePWXb023fY/sXxZ9P296YAIB+ZY7APy/pqr5tN0j6VkQ8X9K3ivsAgDEaWOARcb+kv/ZtvkbSrcXtWyUdqDcWAOwcC0ur2nfoiBaWVgfvPIRRz4E/MyKOS1Lx5zM229H2Qdsd251utzvicgCQ1+Lyuk5GaHF5vdbX3fZfYkbE4YhoR0S71Trjm6AAsOPNz81oytb83EytrzvqV+n/ZHtPRBy3vUfSiTpDAcBOctOB/brpwP7aX3fUI/C7JL2puP0mSV+tJw4AoKwyHyO8XdJ3Jb3A9jHbb5X0YUmvtP0LSa8s7gMAxmjgKZSIeP0mD11ZcxYAwBD4JiYAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BSFDgAJFWpwG2/x/ZDtldt3277iXUFAwBsbeQCt32BpHdJakfEfklTkq6tKxgAYGtVT6FMS3qS7WlJ50r6Q/VIAIAyRi7wiPi9pI9IWpd0XNIjEfHN/v1sH7Tdsd3pdrujJwUAnKbKKZSnSbpG0nMlPVvSk21f179fRByOiHZEtFut1uhJAQCnqXIK5RWSfh0R3Yj4j6Q7Jb20nlgAgEGqFPi6pMtsn2vbkq6UtFZPLADAIFXOgS9LukPSUUk/Ll7rcE25AAADTFd5ckTcKOnGmrIAAIbANzEBICkKHACSosABICkKHMBQFpZWte/QES0srTYd5axHgQMYyuLyuk5GaHF5vekoZz0KHMBQ5udmNGVrfm6m6ShnPUfE2BZrt9vR6XTGth4A7AS2VyKi3b+dI3AASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASIoCB4CkKHAASKpSgds+3/Ydtn9qe832S+oKBgDY2nTF539c0tcj4nW2z5F0bg2ZAAAljFzgtp8q6WWS3ixJEfGYpMfqiQUAGKTKKZTnSepK+pztB2zfbPvJ/TvZPmi7Y7vT7XYrLAcAOFWVAp+WdImkT0fExZL+KemG/p0i4nBEtCOi3Wq1KiwHADhVlQI/JulYRCwX9+9Qr9ABAGMwcoFHxB8l/c72C4pNV0r6SS2pAAADVf0Uyjsl3VZ8AuVXkt5SPRIAoIxKBR4RD0pq1xMFADAMvokJAElR4ACQFAUOAElR4ACQFAUOAElR4ACQFAUOAElR4ACQFAUOAElR4ACQFAUOAElR4ACQFAUOAElR4DjrLSytat+hI1pYWm06CjAUChxnvcXldZ2M0OLyetNRgKFQ4Djrzc/NaMrW/NxM01GAoTgixrZYu92OTqcztvUAYCewvRIRZ1w8hyNwAEiKAgeApChwAEiKAgeApChwAEiKAgeApChwAEiKAgeApChwAEiKAgeApChwAEiKAgeApChwAEiKAgeApCoXuO0p2w/YvruOQACAcuo4Ar9e0loNrwMAGEKlAre9V9KrJd1cTxwAQFlVj8A/Jun9kv672Q62D9ru2O50u92KywEA/m/kArf9GkknImJlq/0i4nBEtCOi3Wq1Rl0OANCnyhH45ZJea/s3kr4k6QrbX6wlFQBgoJELPCIORcTeiJiVdK2kb0fEdbUlAwBsic+BA0BS03W8SETcJ+m+Ol4LAFAOR+AAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeAAkBQFDgBJUeA70MLSqvYdOqKFpdWmowDYRhT4DrS4vK6TEVpcXm86CoBtRIHvQPNzM5qyNT8303QUANvIETG2xdrtdnQ6nbGtBwA7ge2ViGj3b+cIHACSosABICkKHACSosABIKmRC9z2hbbvtb1m+yHb19cZDACwtekKz31c0vsi4qjtp0hasX1PRPykpmwAgC2MfAQeEccj4mhx+++S1iRdUFcwAMDWajkHbntW0sWSljd47KDtju1Ot9utYzkAgGoocNvnSfqKpHdHxKP9j0fE4YhoR0S71WpVXQ4AUKhU4LZ3qVfet0XEnfVEAgCUUeVTKJb0WUlrEfHR+iIBAMqocgR+uaQ3SLrC9oPFz9U15QIADDDyxwgj4juSXGMWAMAQ+CYmACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRFgQNAUhQ4ACRVqcBtX2X7Z7Yftn1DXaEAAIONXOC2pyR9StKrJF0k6fW2L6orGABga1WOwC+V9HBE/CoiHpP0JUnX1BPrdAtLq9p36IgWlla34+UBIKUqBX6BpN+dcv9Yse00tg/a7tjudLvdkRZaXF7XyQgtLq+PlhQAdqAqBe4NtsUZGyIOR0Q7ItqtVmukhebnZjRla35uZqTnA8BONF3hucckXXjK/b2S/lAtzsZuOrBfNx3Yvx0vDQBpVTkC/4Gk59t+ru1zJF0r6a56YgEABhn5CDwiHrf9DknfkDQl6ZaIeKi2ZACALVU5haKIOCLpSE1ZAABD4JuYAJAUBQ4ASVHgAJAUBQ4ASTnijO/ebN9idlfSb0d8+m5Jf64xTl3INRxyDYdcw5nUXFK1bM+JiDO+CTnWAq/Cdici2k3n6Eeu4ZBrOOQazqTmkrYnG6dQACApChwAkspU4IebDrAJcg2HXMMh13AmNZe0DdnSnAMHAJwu0xE4AOAUFDgAJDVxBT7oQsnu+UTx+I9sXzIhuV5u+xHbDxY/HxxDpltsn7C94bXmGpzVoFxjn1Wx7oW277W9Zvsh29dvsM/YZ1YyVxPvryfa/r7tHxa5PrTBPk3Mq0yuRt5jxdpTth+wffcGj9U7r4iYmB/1/lvaX0p6nqRzJP1Q0kV9+1wt6WvqXRHoMknLE5Lr5ZLuHvO8XibpEkmrmzw+9lmVzDX2WRXr7pF0SXH7KZJ+PiHvrzK5mnh/WdJ5xe1dkpYlXTYB8yqTq5H3WLH2eyUtbrR+3fOatCPwMhdKvkbSF6Lne5LOt71nAnKNXUTcL+mvW+zSxKzK5GpERByPiKPF7b9LWtOZ13Ed+8xK5hq7Ygb/KO7uKn76P/XQxLzK5GqE7b2SXi3p5k12qXVek1bgZS6UXOpiyg3kkqSXFP+s+5rtF25zpjKamFVZjc7K9qyki9U7ejtVozPbIpfUwMyK0wEPSjoh6Z6ImIh5lcglNfMe+5ik90v67yaP1zqvSSvwMhdKLnUx5ZqVWfOoev9fwYskfVLS0jZnKqOJWZXR6KxsnyfpK5LeHRGP9j+8wVPGMrMBuRqZWUScjIgXq3fN20tt91+ctpF5lcg19nnZfo2kExGxstVuG2wbeV6TVuBlLpQ8tospD7NmRDz6/3/WRe9KRbts797mXIM0MauBmpyV7V3qleRtEXHnBrs0MrNBuZp+f0XE3yTdJ+mqvocafY9tlquheV0u6bW2f6PeadYrbH+xb59a5zVpBV7mQsl3SXpj8dvcyyQ9EhHHm85l+1m2Xdy+VL3Z/mWbcw3SxKwGampWxZqflbQWER/dZLexz6xMriZmZrtl+/zi9pMkvULST/t2a2JeA3M1Ma+IOBQReyNiVr2O+HZEXNe3W63zqnRNzLrFJhdKtv224vHPqHcNzqslPSzpX5LeMiG5Xifp7bYfl/RvSddG8Wvn7WL7dvV+277b9jFJN6r3C53GZlUy19hnVbhc0hsk/bg4fypJH5A0c0q2JmZWJlcTM9sj6VbbU+oV4Jcj4u6m/z6WzNXUe+wM2zkvvkoPAElN2ikUAEBJFDgAJEWBA0BSFDgAJEWBA0BSFDgAJEWBA0BS/wMI1KzaC3CJUgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 1 2 3 4]\n",
      "[ 0.   2.5  5.   7.5 10. ]\n"
     ]
    }
   ],
   "source": [
    "# Let us create some random data\n",
    "# We create a very simple data set with 5 data items in it. \n",
    "size= 5\n",
    "\n",
    "# mu, sigma = 100, 5000 # mean and standard deviation\n",
    "# error=np.random.normal(mu, sigma, size)\n",
    "\n",
    "x1 = np.arange(0, size)\n",
    "# x2 = np.arange(1, size)\n",
    "\n",
    "# y = 2.5*x1 + error\n",
    "y=2.5 * x1\n",
    "\n",
    "# y = 2*x1 + 10* x2\n",
    "\n",
    "plt.plot(x1, y, 'o', markersize=2)\n",
    "plt.show()\n",
    "\n",
    "print(x1)\n",
    "# print(x2)\n",
    "# print(error)\n",
    "print(y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# If sklearn is not installed \n",
    "# !pip3 install -q sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0]\n",
      " [1]\n",
      " [2]\n",
      " [3]\n",
      " [4]]\n",
      "[ 0.   2.5  5.   7.5 10. ]\n",
      "[2.5]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.linear_model import LinearRegression\n",
    "\n",
    "# X = np.stack([x1, x2], axis=1)\n",
    "X = np.stack([x1], axis=1)\n",
    "print(X)\n",
    "print(y)\n",
    "\n",
    "reg = LinearRegression(fit_intercept=False).fit(X, y)\n",
    "# reg.score(X, y)\n",
    "print(reg.coef_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2.5]\n"
     ]
    }
   ],
   "source": [
    "# print(np.shape(X.T))\n",
    "# print(np.shape(X))\n",
    "\n",
    "# Let use solve this also with the exact linear algebra solution. \n",
    "beta_hat = np.linalg.inv(X.T.dot(X)).dot(X.T).dot(y)\n",
    "\n",
    "print(beta_hat)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 beta =  0  Cost= 187.5\n",
      "1 beta =  0.15  Cost= 165.675\n",
      "2 beta =  0.29100000000000004  Cost= 146.39043\n",
      "3 beta =  0.4235400000000001  Cost= 129.350583948\n",
      "4 beta =  0.5481276000000002  Cost= 114.29417597645278\n",
      "5 beta =  0.6652399440000002  Cost= 100.99033389279367\n",
      "6 beta =  0.7753255473600003  Cost= 89.23505902767249\n",
      "7 beta =  0.8788060145184002  Cost= 78.84809815685139\n",
      "8 beta =  0.9760776536472963  Cost= 69.67017953139388\n",
      "9 beta =  1.0675129944284585  Cost= 61.56057063393965\n",
      "10 beta =  1.153462214762751  Cost= 54.394920212149074\n",
      "11 beta =  1.2342544818769858  Cost= 48.063351499454924\n",
      "12 beta =  1.3101992129643667  Cost= 42.46877738491837\n",
      "13 beta =  1.3815872601865047  Cost= 37.52541169731388\n",
      "14 beta =  1.4486920245753143  Cost= 33.157453775746546\n",
      "15 beta =  1.5117705031007955  Cost= 29.29792615624964\n",
      "16 beta =  1.5710642729147477  Cost= 25.887647551662187\n",
      "17 beta =  1.6268004165398628  Cost= 22.874325376648713\n",
      "18 beta =  1.6791923915474711  Cost= 20.211753902806798\n",
      "19 beta =  1.728440848054623  Cost= 17.859105748520083\n",
      "20 beta =  1.7747343971713456  Cost= 15.78030583939234\n",
      "21 beta =  1.818250333341065  Cost= 13.943478239687074\n",
      "22 beta =  1.859155313340601  Cost= 12.320457372587502\n",
      "23 beta =  1.8976059945401649  Cost= 10.886356134418314\n",
      "24 beta =  1.933749634867755  Cost= 9.619184280372027\n",
      "25 beta =  1.9677246567756896  Cost= 8.499511230136722\n",
      "26 beta =  1.9996611773691482  Cost= 7.51016812294881\n",
      "27 beta =  2.029681506726999  Cost= 6.635984553437574\n",
      "28 beta =  2.057900616323379  Cost= 5.8635559514174425\n",
      "29 beta =  2.0844265793439765  Cost= 5.1810380386724475\n",
      "30 beta =  2.109360984583338  Cost= 4.57796521097097\n",
      "31 beta =  2.132799325508338  Cost= 4.045090060413946\n",
      "32 beta =  2.1548313659778375  Cost= 3.574241577381767\n",
      "33 beta =  2.1755414840191674  Cost= 3.158199857774526\n",
      "34 beta =  2.1950089949780174  Cost= 2.7905853943295718\n",
      "35 beta =  2.2133084552793365  Cost= 2.465761254429606\n",
      "36 beta =  2.2305099479625765  Cost= 2.1787466444139967\n",
      "37 beta =  2.246679351084822  Cost= 1.9251405350042066\n",
      "38 beta =  2.2618785900197325  Cost= 1.701054176729719\n",
      "39 beta =  2.2761658746185485  Cost= 1.5030514705583815\n",
      "40 beta =  2.2895959221414355  Cost= 1.3280962793853863\n",
      "41 beta =  2.3022201668129494  Cost= 1.173505872464927\n",
      "42 beta =  2.3140869568041724  Cost= 1.0369097889100094\n",
      "43 beta =  2.325241739395922  Cost= 0.9162134894808864\n",
      "44 beta =  2.3357272350321665  Cost= 0.8095662393053119\n",
      "45 beta =  2.3455836009302367  Cost= 0.7153327290501723\n",
      "46 beta =  2.3548485848744223  Cost= 0.6320679993887337\n",
      "47 beta =  2.363557669781957  Cost= 0.5584952842598836\n",
      "48 beta =  2.3717442095950396  Cost= 0.49348643317203383\n",
      "49 beta =  2.3794395570193374  Cost= 0.4360446123508076\n",
      "50 beta =  2.386673183598177  Cost= 0.3852890194731736\n",
      "51 beta =  2.3934727925822865  Cost= 0.34044137760649684\n",
      "52 beta =  2.3998644250273493  Cost= 0.3008140012531004\n",
      "53 beta =  2.4058725595257084  Cost= 0.26579925150723877\n",
      "54 beta =  2.411520205954166  Cost= 0.2348602186317958\n",
      "55 beta =  2.416828993596916  Cost= 0.20752248918305471\n",
      "56 beta =  2.421819253981101  Cost= 0.1833668714421468\n",
      "57 beta =  2.426510098742235  Cost= 0.16202296760628054\n",
      "58 beta =  2.430919492817701  Cost= 0.1431634941769091\n",
      "59 beta =  2.435064323248639  Cost= 0.12649926345471704\n",
      "60 beta =  2.4389604638537206  Cost= 0.11177474918858862\n",
      "61 beta =  2.4426228360224975  Cost= 0.09876416838303623\n",
      "62 beta =  2.446065465861148  Cost= 0.08726801918325036\n",
      "63 beta =  2.449301537909479  Cost= 0.07711002175032013\n",
      "64 beta =  2.4523434456349102  Cost= 0.06813441521858266\n",
      "65 beta =  2.4552028388968155  Cost= 0.06020356928714\n",
      "66 beta =  2.4578906685630066  Cost= 0.05319587382211671\n",
      "67 beta =  2.4604172284492263  Cost= 0.04700387410922224\n",
      "68 beta =  2.462792194742273  Cost= 0.04153262316290836\n",
      "69 beta =  2.4650246630577364  Cost= 0.03669822582674612\n",
      "70 beta =  2.467123183274272  Cost= 0.032426552340512935\n",
      "71 beta =  2.4690957922778156  Cost= 0.02865210164807771\n",
      "72 beta =  2.4709500447411465  Cost= 0.025316997016241547\n",
      "73 beta =  2.4726930420566777  Cost= 0.02237009856355114\n",
      "74 beta =  2.474331459533277  Cost= 0.019766219090753746\n",
      "75 beta =  2.4758715719612803  Cost= 0.01746543118859023\n",
      "76 beta =  2.4773192776436033  Cost= 0.015432454998238695\n",
      "77 beta =  2.478680120984987  Cost= 0.01363611723644358\n",
      "78 beta =  2.479959313725888  Cost= 0.012048873190121483\n",
      "79 beta =  2.4811617549023346  Cost= 0.010646384350791443\n",
      "80 beta =  2.4822920496081946  Cost= 0.009407145212359178\n",
      "81 beta =  2.483354526631703  Cost= 0.008312153509640565\n",
      "82 beta =  2.484353255033801  Cost= 0.00734461884111842\n",
      "83 beta =  2.4852920597317727  Cost= 0.006489705208012267\n",
      "84 beta =  2.486174536147866  Cost= 0.005734303521799706\n",
      "85 beta =  2.4870040639789943  Cost= 0.005066830591862191\n",
      "86 beta =  2.4877838201402547  Cost= 0.004477051510969419\n",
      "87 beta =  2.4885167909318393  Cost= 0.00395592271509267\n",
      "88 beta =  2.489205783475929  Cost= 0.0034954533110557644\n",
      "89 beta =  2.4898534364673734  Cost= 0.0030885825456488103\n",
      "90 beta =  2.490462230279331  Cost= 0.0027290715373352223\n",
      "91 beta =  2.491034496462571  Cost= 0.002411407610389568\n",
      "92 beta =  2.491572426674817  Cost= 0.0021307197645401158\n",
      "93 beta =  2.492078081074328  Cost= 0.001882703983947638\n",
      "94 beta =  2.492553396209868  Cost= 0.0016635572402162084\n",
      "95 beta =  2.4930001924372758  Cost= 0.0014699191774551193\n",
      "96 beta =  2.493420180891039  Cost= 0.0012988205851994649\n",
      "97 beta =  2.4938149700375765  Cost= 0.001147637869082291\n",
      "98 beta =  2.4941860718353217  Cost= 0.001014052821121186\n",
      "99 beta =  2.4945349075252023  Cost= 0.0008960170727426972\n"
     ]
    }
   ],
   "source": [
    "# Now, we do gradient descent here with a very simple numpy array \n",
    "\n",
    "learningRate = 0.01\n",
    "num_iteration = 100\n",
    "beta=0\n",
    "\n",
    "n = float(size)\n",
    "# print(\"Sample size\", n)\n",
    "\n",
    "# Let's start with main iterative part of gradient descent algorithm \n",
    "for i in range(num_iteration):\n",
    "    \n",
    "    # Calculate the prediction with current regression coefficients. \n",
    "    y_prediction = beta * x1 \n",
    "    \n",
    "    \n",
    "    # We compute costs just for monitoring \n",
    "    cost= sum (( y - y_prediction)**2)\n",
    "\n",
    "    # calculate gradients. \n",
    "    m_gradient = (-1.0/n) * sum (x1 * (y - y_prediction) )\n",
    "    \n",
    "    \n",
    "    print(i , \"beta = \", beta, \" Cost=\", cost)\n",
    "        \n",
    "    # update the weights - Regression Coefficients \n",
    "    beta = beta - learningRate * m_gradient "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}