05. Статистика.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "число точек 204\n",
      "максимальное значение 100\n",
      "минимальное значение 1\n",
      "следующий минимум 1\n",
      "следующий максимум 49\n",
      "mean(num_friends) 7.333333333333333\n",
      "median(num_friends) 6.0\n",
      "quantile(num_friends, 0.10) 1\n",
      "quantile(num_friends, 0.25) 3\n",
      "quantile(num_friends, 0.75) 9\n",
      "quantile(num_friends, 0.90) 13\n",
      "mode(num_friends) [1, 6]\n",
      "data_range(num_friends) 99\n",
      "variance(num_friends) 81.54351395730716\n",
      "standard_deviation(num_friends) 9.03014473623248\n",
      "interquartile_range(num_friends) 6\n",
      "covariance(num_friends, daily_minutes) 22.425435139573064\n",
      "correlation(num_friends, daily_minutes) 0.24736957366478218\n",
      "correlation(num_friends_good, daily_minutes_good) 0.5736792115665573\n"
     ]
    }
   ],
   "source": [
    "# -*- coding: utf-8 -*-\n",
    "\n",
    "# statistics.py\n",
    "\n",
    "import sys\n",
    "sys.path.append(\"../code-python3-ru\")\n",
    "\n",
    "from collections import Counter\n",
    "from lib.linear_algebra import sum_of_squares, dot\n",
    "import math\n",
    "\n",
    "# число друзей\n",
    "num_friends = [100,49,41,40,25,21,21,19,19,18,\n",
    "               18,16,15,15,15,15,14,14,13,13,\n",
    "               13,13,12,12,11,10,10,10,10,10,\n",
    "               10,10,10,10,10,10,10,10,10,10,\n",
    "               9,9,9,9,9,9,9,9,9,9,\n",
    "               9,9,9,9,9,9,9,9,8,8,\n",
    "               8,8,8,8,8,8,8,8,8,8,\n",
    "               8,7,7,7,7,7,7,7,7,7,\n",
    "               7,7,7,7,7,7,6,6,6,6,\n",
    "               6,6,6,6,6,6,6,6,6,6,\n",
    "               6,6,6,6,6,6,6,6,5,5,\n",
    "               5,5,5,5,5,5,5,5,5,5,\n",
    "               5,5,5,5,5,4,4,4,4,4,\n",
    "               4,4,4,4,4,4,4,4,4,4,\n",
    "               4,4,4,4,4,3,3,3,3,3,\n",
    "               3,3,3,3,3,3,3,3,3,3,\n",
    "               3,3,3,3,3,2,2,2,2,2,\n",
    "               2,2,2,2,2,2,2,2,2,2,\n",
    "               2,2,1,1,1,1,1,1,1,1,\n",
    "               1,1,1,1,1,1,1,1,1,1,\n",
    "               1,1,1,1]\n",
    "\n",
    "def make_friend_counts_histogram(plt):\n",
    "    friend_counts = Counter(num_friends)\n",
    "    xs = range(101)\n",
    "    ys = [friend_counts[x] for x in xs]\n",
    "    plt.bar(xs, ys)\n",
    "    plt.axis([0,101,0,25])\n",
    "    plt.title(\"Гистограмма количества друзей\")\n",
    "    plt.xlabel(\"Количество друзей\")\n",
    "    plt.ylabel(\"Количество людей\")\n",
    "    plt.show()\n",
    "\n",
    "num_points = len(num_friends)               # 204\n",
    "\n",
    "largest_value = max(num_friends)            # 100\n",
    "smallest_value = min(num_friends)           # 1\n",
    "\n",
    "sorted_values = sorted(num_friends)\n",
    "smallest_value = sorted_values[0]           # 1\n",
    "second_smallest_value = sorted_values[1]    # 1\n",
    "second_largest_value = sorted_values[-2]    # 49\n",
    "\n",
    "# в Python 2 частное округляется вниз, поэтому следует импортировать\n",
    "# from __future__ import division\n",
    "def mean(x):\n",
    "    return sum(x) / len(x)\n",
    "\n",
    "def median(v):\n",
    "    \"\"\"возвращает ближайшее к середине значение для v\"\"\"\n",
    "    n = len(v)\n",
    "    sorted_v = sorted(v)\n",
    "    midpoint = n // 2    # индекс серединного значения\n",
    "\n",
    "    if n % 2 == 1:\n",
    "        # если нечетное, вернуть серединное значение\n",
    "        return sorted_v[midpoint]\n",
    "    else:\n",
    "        # если четное, вернуть среднее 2-х серединных значений\n",
    "        lo = midpoint - 1\n",
    "        hi = midpoint\n",
    "        return (sorted_v[lo] + sorted_v[hi]) / 2\n",
    "\n",
    "def quantile(x, p):\n",
    "    \"\"\"возвращает значение в x, соответствующее p-ому проценту данных\"\"\"\n",
    "    p_index = int(p * len(x))\n",
    "    return sorted(x)[p_index]\n",
    "\n",
    "def mode(x):\n",
    "    \"\"\"возвращает список, поскольку мод может быть больше одной\"\"\"\n",
    "    counts = Counter(x)\n",
    "    max_count = max(counts.values())\n",
    "    return [x_i for x_i, count in counts.items()\n",
    "            if count == max_count]\n",
    "\n",
    "# слово \"range\" в Python уже используется, поэтому берем другое\n",
    "def data_range(x):\n",
    "    return max(x) - min(x)\n",
    "\n",
    "def de_mean(x):\n",
    "    \"\"\"пересчитать x, вычтя его среднее (среднее результата будет = 0)\"\"\"\n",
    "    x_bar = mean(x)\n",
    "    return [x_i - x_bar for x_i in x]\n",
    "\n",
    "def variance(x):\n",
    "    \"\"\"предполагается, что вектор x состоит как минимум из 2х элементов\"\"\"\n",
    "    n = len(x)\n",
    "    deviations = de_mean(x)\n",
    "    return sum_of_squares(deviations) / (n - 1)\n",
    "\n",
    "def standard_deviation(x):\n",
    "    return math.sqrt(variance(x))\n",
    "\n",
    "def interquartile_range(x):\n",
    "    return quantile(x, 0.75) - quantile(x, 0.25)\n",
    "\n",
    "####\n",
    "#\n",
    "# КОРРЕЛЯЦИЯ\n",
    "#\n",
    "#####\n",
    "\n",
    "# ежедневные минуты - сколько минут в день каждый пользователь тратит на DataSciencester\n",
    "daily_minutes = [1,68.77,51.25,52.08,38.36,44.54,57.13,51.4,41.42,31.22,\n",
    "                 34.76,54.01,38.79,47.59,49.1,27.66,41.03,36.73,48.65,28.12,\n",
    "                 46.62,35.57,32.98,35,26.07,23.77,39.73,40.57,31.65,31.21,\n",
    "                 36.32,20.45,21.93,26.02,27.34,23.49,46.94,30.5,33.8,24.23,\n",
    "                 21.4,27.94,32.24,40.57,25.07,19.42,22.39,18.42,46.96,23.72,\n",
    "                 26.41,26.97,36.76,40.32,35.02,29.47,30.2,31,38.11,38.18,\n",
    "                 36.31,21.03,30.86,36.07,28.66,29.08,37.28,15.28,24.17,22.31,\n",
    "                 30.17,25.53,19.85,35.37,44.6,17.23,13.47,26.33,35.02,32.09,\n",
    "                 24.81,19.33,28.77,24.26,31.98,25.73,24.86,16.28,34.51,15.23,\n",
    "                 39.72,40.8,26.06,35.76,34.76,16.13,44.04,18.03,19.65,32.62,\n",
    "                 35.59,39.43,14.18,35.24,40.13,41.82,35.45,36.07,43.67,24.61,\n",
    "                 20.9,21.9,18.79,27.61,27.21,26.61,29.77,20.59,27.53,13.82,\n",
    "                 33.2,25,33.1,36.65,18.63,14.87,22.2,36.81,25.53,24.62,\n",
    "                 26.25,18.21,28.08,19.42,29.79,32.8,35.99,28.32,27.79,35.88,\n",
    "                 29.06,36.28,14.1,36.63,37.49,26.9,18.58,38.48,24.48,18.95,\n",
    "                 33.55,14.24,29.04,32.51,25.63,22.22,19,32.73,15.16,13.9,\n",
    "                 27.2,32.01,29.27,33,13.74,20.42,27.32,18.23,35.35,28.48,\n",
    "                 9.08,24.62,20.12,35.26,19.92,31.02,16.49,12.16,30.7,31.22,\n",
    "                 34.65,13.13,27.51,33.2,31.57,14.1,33.42,17.44,10.12,24.42,\n",
    "                 9.82,23.39,30.93,15.03,21.67,31.09,33.29,22.61,26.89,23.48,\n",
    "                 8.38,27.81,32.35,23.84]\n",
    "\n",
    "def covariance(x, y):\n",
    "    n = len(x)\n",
    "    return dot(de_mean(x), de_mean(y)) / (n - 1)\n",
    "\n",
    "def correlation(x, y):\n",
    "    stdev_x = standard_deviation(x)\n",
    "    stdev_y = standard_deviation(y)\n",
    "    if stdev_x > 0 and stdev_y > 0:\n",
    "        return covariance(x, y) / stdev_x / stdev_y\n",
    "    else:\n",
    "        return 0 # если переменные не меняются, то корреляция равна нулю\n",
    "\n",
    "outlier = num_friends.index(100) # индекс выброса\n",
    "\n",
    "num_friends_good = [x\n",
    "                    for i, x in enumerate(num_friends)\n",
    "                    if i != outlier]\n",
    "\n",
    "daily_minutes_good = [x\n",
    "                      for i, x in enumerate(daily_minutes)\n",
    "                      if i != outlier]\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "\n",
    "    print(\"число точек\", len(num_friends))\n",
    "    print(\"максимальное значение\", max(num_friends))\n",
    "    print(\"минимальное значение\", min(num_friends))\n",
    "    print(\"следующий минимум\", sorted_values[1])\n",
    "    print(\"следующий максимум\", sorted_values[-2]  )\n",
    "    print(\"mean(num_friends)\", mean(num_friends))\n",
    "    print(\"median(num_friends)\", median(num_friends))\n",
    "    print(\"quantile(num_friends, 0.10)\", quantile(num_friends, 0.10))\n",
    "    print(\"quantile(num_friends, 0.25)\", quantile(num_friends, 0.25))\n",
    "    print(\"quantile(num_friends, 0.75)\", quantile(num_friends, 0.75))\n",
    "    print(\"quantile(num_friends, 0.90)\", quantile(num_friends, 0.90))\n",
    "    print(\"mode(num_friends)\", mode(num_friends))\n",
    "    print(\"data_range(num_friends)\", data_range(num_friends))\n",
    "    print(\"variance(num_friends)\", variance(num_friends))\n",
    "    print(\"standard_deviation(num_friends)\", standard_deviation(num_friends))\n",
    "    print(\"interquartile_range(num_friends)\", interquartile_range(num_friends))\n",
    "\n",
    "    print(\"covariance(num_friends, daily_minutes)\", covariance(num_friends, daily_minutes))\n",
    "    print(\"correlation(num_friends, daily_minutes)\", correlation(num_friends, daily_minutes))\n",
    "    print(\"correlation(num_friends_good, daily_minutes_good)\", correlation(num_friends_good, daily_minutes_good))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}