{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "tags": [
     "remove_input"
    ]
   },
   "outputs": [],
   "source": [
    "path_data = '../../data/'\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import math\n",
    "import scipy.stats as stats\n",
    "\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "from mpl_toolkits.mplot3d import Axes3D\n",
    "plt.style.use('fivethirtyeight')\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "tags": [
     "remove_input"
    ]
   },
   "outputs": [],
   "source": [
    "def distance(point1, point2):\n",
    "    \"\"\"Returns the distance between point1 and point2\n",
    "    where each argument is an array \n",
    "    consisting of the coordinates of the point\"\"\"\n",
    "    return np.sqrt(np.sum((point1 - point2)**2))\n",
    "\n",
    "def all_distances(training, new_point):\n",
    "    \"\"\"Returns an array of distances\n",
    "    #between each point in the training set\n",
    "    #and the new point (which is a row of attributes)\"\"\"\n",
    "    attributes = training.drop(columns=['Class'])\n",
    "    def distance_from_point(row):\n",
    "        return distance(new_point, np.array(row)) # --- possible issue as original = np.array(new_point)\n",
    "    return attributes.apply(distance_from_point, axis=1)\n",
    "\n",
    "def table_with_distances(training, new_point):\n",
    "    \"\"\"Augments the training table \n",
    "    with a column of distances from new_point\"\"\"\n",
    "    training1 = training.copy()\n",
    "    training1['Distance'] = all_distances(training1, new_point)\n",
    "    return training1\n",
    "\n",
    "def closest(training, new_point, k):\n",
    "    \"\"\"Returns a table of the k rows of the augmented table\n",
    "    corresponding to the k smallest distances\"\"\"\n",
    "    with_dists = table_with_distances(training, new_point)\n",
    "    sorted_by_distance = with_dists.sort_values(by=['Distance'])\n",
    "    topk = sorted_by_distance.take(np.arange(k))\n",
    "    return topk\n",
    "\n",
    "def majority(topkclasses):\n",
    "    ones = len(topkclasses[topkclasses['Class'] == 1])\n",
    "    zeros = len(topkclasses[topkclasses['Class'] == 0])\n",
    "    if ones > zeros:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "def classify(training, new_point, k):\n",
    "    closestk = closest(training, new_point, k)\n",
    "    topkclasses = closestk[['Class']]\n",
    "    return majority(topkclasses)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "tags": [
     "remove_input"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Alcohol</th>\n",
       "      <th>Malic Acid</th>\n",
       "      <th>Ash</th>\n",
       "      <th>Alcalinity of Ash</th>\n",
       "      <th>Magnesium</th>\n",
       "      <th>Total Phenols</th>\n",
       "      <th>Flavanoids</th>\n",
       "      <th>Nonflavanoid phenols</th>\n",
       "      <th>Proanthocyanins</th>\n",
       "      <th>Color Intensity</th>\n",
       "      <th>Hue</th>\n",
       "      <th>OD280/OD315 of diulted wines</th>\n",
       "      <th>Proline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>14.23</td>\n",
       "      <td>1.71</td>\n",
       "      <td>2.43</td>\n",
       "      <td>15.6</td>\n",
       "      <td>127</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.06</td>\n",
       "      <td>0.28</td>\n",
       "      <td>2.29</td>\n",
       "      <td>5.64</td>\n",
       "      <td>1.04</td>\n",
       "      <td>3.92</td>\n",
       "      <td>1065</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>13.20</td>\n",
       "      <td>1.78</td>\n",
       "      <td>2.14</td>\n",
       "      <td>11.2</td>\n",
       "      <td>100</td>\n",
       "      <td>2.65</td>\n",
       "      <td>2.76</td>\n",
       "      <td>0.26</td>\n",
       "      <td>1.28</td>\n",
       "      <td>4.38</td>\n",
       "      <td>1.05</td>\n",
       "      <td>3.40</td>\n",
       "      <td>1050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>13.16</td>\n",
       "      <td>2.36</td>\n",
       "      <td>2.67</td>\n",
       "      <td>18.6</td>\n",
       "      <td>101</td>\n",
       "      <td>2.80</td>\n",
       "      <td>3.24</td>\n",
       "      <td>0.30</td>\n",
       "      <td>2.81</td>\n",
       "      <td>5.68</td>\n",
       "      <td>1.03</td>\n",
       "      <td>3.17</td>\n",
       "      <td>1185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>14.37</td>\n",
       "      <td>1.95</td>\n",
       "      <td>2.50</td>\n",
       "      <td>16.8</td>\n",
       "      <td>113</td>\n",
       "      <td>3.85</td>\n",
       "      <td>3.49</td>\n",
       "      <td>0.24</td>\n",
       "      <td>2.18</td>\n",
       "      <td>7.80</td>\n",
       "      <td>0.86</td>\n",
       "      <td>3.45</td>\n",
       "      <td>1480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>13.24</td>\n",
       "      <td>2.59</td>\n",
       "      <td>2.87</td>\n",
       "      <td>21.0</td>\n",
       "      <td>118</td>\n",
       "      <td>2.80</td>\n",
       "      <td>2.69</td>\n",
       "      <td>0.39</td>\n",
       "      <td>1.82</td>\n",
       "      <td>4.32</td>\n",
       "      <td>1.04</td>\n",
       "      <td>2.93</td>\n",
       "      <td>735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class  Alcohol  Malic Acid   Ash  Alcalinity of Ash  Magnesium  \\\n",
       "0      1    14.23        1.71  2.43               15.6        127   \n",
       "1      1    13.20        1.78  2.14               11.2        100   \n",
       "2      1    13.16        2.36  2.67               18.6        101   \n",
       "3      1    14.37        1.95  2.50               16.8        113   \n",
       "4      1    13.24        2.59  2.87               21.0        118   \n",
       "\n",
       "   Total Phenols  Flavanoids  Nonflavanoid phenols  Proanthocyanins  \\\n",
       "0           2.80        3.06                  0.28             2.29   \n",
       "1           2.65        2.76                  0.26             1.28   \n",
       "2           2.80        3.24                  0.30             2.81   \n",
       "3           3.85        3.49                  0.24             2.18   \n",
       "4           2.80        2.69                  0.39             1.82   \n",
       "\n",
       "   Color Intensity   Hue  OD280/OD315 of diulted wines  Proline  \n",
       "0             5.64  1.04                          3.92     1065  \n",
       "1             4.38  1.05                          3.40     1050  \n",
       "2             5.68  1.03                          3.17     1185  \n",
       "3             7.80  0.86                          3.45     1480  \n",
       "4             4.32  1.04                          2.93      735  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wine = pd.read_csv(path_data + 'wine.csv')\n",
    "\n",
    "# For converting Class to binary\n",
    "\n",
    "def is_one(x):\n",
    "    if x == 1:\n",
    "        return 1\n",
    "    else:\n",
    "        return 0\n",
    "\n",
    "wine['Class1'] = wine['Class'].apply(is_one)\n",
    "\n",
    "# This creates a column 'Class1' - we drop the 'Class' column, rename 'Class1' as 'Class'\n",
    "# then move the column to the first position using 'pop' and 'insert'\n",
    "\n",
    "wine = wine.drop(columns=['Class'])\n",
    "wine = wine.rename(columns={'Class1': 'Class'})\n",
    "class_label = wine.pop('Class')\n",
    "wine.insert(0, 'Class', class_label)\n",
    "wine.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Accuracy of Classifier\n",
    "To see how well our classifier does, we might put 50% of the data into the training set and the other 50% into the test set.  Basically, we are setting aside some data for later use, so we can use it to measure the accuracy of our classifier.  We've been calling that the *test set*. Sometimes people will call the data that you set aside for testing a *hold-out set*, and they'll call this strategy for estimating accuracy the *hold-out method*.\n",
    "\n",
    "Note that this approach requires great discipline.  Before you start applying machine learning methods, you have to take some of your data and set it aside for testing.  You must avoid using the test set for developing your classifier: you shouldn't use it to help train your classifier or tweak its settings or for brainstorming ways to improve your classifier.  Instead, you should use it only once, at the very end, after you've finalized your classifier, when you want an unbiased estimate of its accuracy."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Measuring the Accuracy of Our Wine Classifier\n",
    "OK, so let's apply the hold-out method to evaluate the effectiveness of the $k$-nearest neighbor classifier for identifying wines.  The data set has 178 wines, so we'll randomly permute the data set and put 89 of them in the training set and the remaining 89 in the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Alcohol</th>\n",
       "      <th>Malic Acid</th>\n",
       "      <th>Ash</th>\n",
       "      <th>Alcalinity of Ash</th>\n",
       "      <th>Magnesium</th>\n",
       "      <th>Total Phenols</th>\n",
       "      <th>Flavanoids</th>\n",
       "      <th>Nonflavanoid phenols</th>\n",
       "      <th>Proanthocyanins</th>\n",
       "      <th>Color Intensity</th>\n",
       "      <th>Hue</th>\n",
       "      <th>OD280/OD315 of diulted wines</th>\n",
       "      <th>Proline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>0</td>\n",
       "      <td>12.96</td>\n",
       "      <td>3.45</td>\n",
       "      <td>2.35</td>\n",
       "      <td>18.5</td>\n",
       "      <td>106</td>\n",
       "      <td>1.39</td>\n",
       "      <td>0.70</td>\n",
       "      <td>0.40</td>\n",
       "      <td>0.94</td>\n",
       "      <td>5.28</td>\n",
       "      <td>0.68</td>\n",
       "      <td>1.75</td>\n",
       "      <td>675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>94</th>\n",
       "      <td>0</td>\n",
       "      <td>11.62</td>\n",
       "      <td>1.99</td>\n",
       "      <td>2.28</td>\n",
       "      <td>18.0</td>\n",
       "      <td>98</td>\n",
       "      <td>3.02</td>\n",
       "      <td>2.26</td>\n",
       "      <td>0.17</td>\n",
       "      <td>1.35</td>\n",
       "      <td>3.25</td>\n",
       "      <td>1.16</td>\n",
       "      <td>2.96</td>\n",
       "      <td>345</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38</th>\n",
       "      <td>1</td>\n",
       "      <td>13.07</td>\n",
       "      <td>1.50</td>\n",
       "      <td>2.10</td>\n",
       "      <td>15.5</td>\n",
       "      <td>98</td>\n",
       "      <td>2.40</td>\n",
       "      <td>2.64</td>\n",
       "      <td>0.28</td>\n",
       "      <td>1.37</td>\n",
       "      <td>3.70</td>\n",
       "      <td>1.18</td>\n",
       "      <td>2.69</td>\n",
       "      <td>1020</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>113</th>\n",
       "      <td>0</td>\n",
       "      <td>11.41</td>\n",
       "      <td>0.74</td>\n",
       "      <td>2.50</td>\n",
       "      <td>21.0</td>\n",
       "      <td>88</td>\n",
       "      <td>2.48</td>\n",
       "      <td>2.01</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.44</td>\n",
       "      <td>3.08</td>\n",
       "      <td>1.10</td>\n",
       "      <td>2.31</td>\n",
       "      <td>434</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>1</td>\n",
       "      <td>14.12</td>\n",
       "      <td>1.48</td>\n",
       "      <td>2.32</td>\n",
       "      <td>16.8</td>\n",
       "      <td>95</td>\n",
       "      <td>2.20</td>\n",
       "      <td>2.43</td>\n",
       "      <td>0.26</td>\n",
       "      <td>1.57</td>\n",
       "      <td>5.00</td>\n",
       "      <td>1.17</td>\n",
       "      <td>2.82</td>\n",
       "      <td>1280</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Class  Alcohol  Malic Acid   Ash  Alcalinity of Ash  Magnesium  \\\n",
       "163      0    12.96        3.45  2.35               18.5        106   \n",
       "94       0    11.62        1.99  2.28               18.0         98   \n",
       "38       1    13.07        1.50  2.10               15.5         98   \n",
       "113      0    11.41        0.74  2.50               21.0         88   \n",
       "11       1    14.12        1.48  2.32               16.8         95   \n",
       "\n",
       "     Total Phenols  Flavanoids  Nonflavanoid phenols  Proanthocyanins  \\\n",
       "163           1.39        0.70                  0.40             0.94   \n",
       "94            3.02        2.26                  0.17             1.35   \n",
       "38            2.40        2.64                  0.28             1.37   \n",
       "113           2.48        2.01                  0.42             1.44   \n",
       "11            2.20        2.43                  0.26             1.57   \n",
       "\n",
       "     Color Intensity   Hue  OD280/OD315 of diulted wines  Proline  \n",
       "163             5.28  0.68                          1.75      675  \n",
       "94              3.25  1.16                          2.96      345  \n",
       "38              3.70  1.18                          2.69     1020  \n",
       "113             3.08  1.10                          2.31      434  \n",
       "11              5.00  1.17                          2.82     1280  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shuffled_wine = wine.sample(len(wine), replace=False) \n",
    "training_set = shuffled_wine.take(np.arange(89))\n",
    "training_set.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Alcohol</th>\n",
       "      <th>Malic Acid</th>\n",
       "      <th>Ash</th>\n",
       "      <th>Alcalinity of Ash</th>\n",
       "      <th>Magnesium</th>\n",
       "      <th>Total Phenols</th>\n",
       "      <th>Flavanoids</th>\n",
       "      <th>Nonflavanoid phenols</th>\n",
       "      <th>Proanthocyanins</th>\n",
       "      <th>Color Intensity</th>\n",
       "      <th>Hue</th>\n",
       "      <th>OD280/OD315 of diulted wines</th>\n",
       "      <th>Proline</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>129</th>\n",
       "      <td>0</td>\n",
       "      <td>12.04</td>\n",
       "      <td>4.30</td>\n",
       "      <td>2.38</td>\n",
       "      <td>22.0</td>\n",
       "      <td>80</td>\n",
       "      <td>2.10</td>\n",
       "      <td>1.75</td>\n",
       "      <td>0.42</td>\n",
       "      <td>1.35</td>\n",
       "      <td>2.60</td>\n",
       "      <td>0.79</td>\n",
       "      <td>2.57</td>\n",
       "      <td>580</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>0</td>\n",
       "      <td>12.37</td>\n",
       "      <td>1.07</td>\n",
       "      <td>2.10</td>\n",
       "      <td>18.5</td>\n",
       "      <td>88</td>\n",
       "      <td>3.52</td>\n",
       "      <td>3.75</td>\n",
       "      <td>0.24</td>\n",
       "      <td>1.95</td>\n",
       "      <td>4.50</td>\n",
       "      <td>1.04</td>\n",
       "      <td>2.77</td>\n",
       "      <td>660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>1</td>\n",
       "      <td>14.75</td>\n",
       "      <td>1.73</td>\n",
       "      <td>2.39</td>\n",
       "      <td>11.4</td>\n",
       "      <td>91</td>\n",
       "      <td>3.10</td>\n",
       "      <td>3.69</td>\n",
       "      <td>0.43</td>\n",
       "      <td>2.81</td>\n",
       "      <td>5.40</td>\n",
       "      <td>1.25</td>\n",
       "      <td>2.73</td>\n",
       "      <td>1150</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>1</td>\n",
       "      <td>14.06</td>\n",
       "      <td>2.15</td>\n",
       "      <td>2.61</td>\n",
       "      <td>17.6</td>\n",
       "      <td>121</td>\n",
       "      <td>2.60</td>\n",
       "      <td>2.51</td>\n",
       "      <td>0.31</td>\n",
       "      <td>1.25</td>\n",
       "      <td>5.05</td>\n",
       "      <td>1.06</td>\n",
       "      <td>3.58</td>\n",
       "      <td>1295</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69</th>\n",
       "      <td>0</td>\n",
       "      <td>12.21</td>\n",
       "      <td>1.19</td>\n",
       "      <td>1.75</td>\n",
       "      <td>16.8</td>\n",
       "      <td>151</td>\n",
       "      <td>1.85</td>\n",
       "      <td>1.28</td>\n",
       "      <td>0.14</td>\n",
       "      <td>2.50</td>\n",
       "      <td>2.85</td>\n",
       "      <td>1.28</td>\n",
       "      <td>3.07</td>\n",
       "      <td>718</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Class  Alcohol  Malic Acid   Ash  Alcalinity of Ash  Magnesium  \\\n",
       "129      0    12.04        4.30  2.38               22.0         80   \n",
       "98       0    12.37        1.07  2.10               18.5         88   \n",
       "13       1    14.75        1.73  2.39               11.4         91   \n",
       "7        1    14.06        2.15  2.61               17.6        121   \n",
       "69       0    12.21        1.19  1.75               16.8        151   \n",
       "\n",
       "     Total Phenols  Flavanoids  Nonflavanoid phenols  Proanthocyanins  \\\n",
       "129           2.10        1.75                  0.42             1.35   \n",
       "98            3.52        3.75                  0.24             1.95   \n",
       "13            3.10        3.69                  0.43             2.81   \n",
       "7             2.60        2.51                  0.31             1.25   \n",
       "69            1.85        1.28                  0.14             2.50   \n",
       "\n",
       "     Color Intensity   Hue  OD280/OD315 of diulted wines  Proline  \n",
       "129             2.60  0.79                          2.57      580  \n",
       "98              4.50  1.04                          2.77      660  \n",
       "13              5.40  1.25                          2.73     1150  \n",
       "7               5.05  1.06                          3.58     1295  \n",
       "69              2.85  1.28                          3.07      718  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set  = shuffled_wine.take(np.arange(89, 178))\n",
    "test_set.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We'll train the classifier using the 89 wines in the training set, and evaluate how well it performs on the test set. To make our lives easier, we'll write a function to evaluate a classifier on every wine in the test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_zero(array):\n",
    "    \"\"\"Counts the number of 0's in an array\"\"\"\n",
    "    return len(array) - np.count_nonzero(array)\n",
    "\n",
    "def count_equal(array1, array2):\n",
    "    \"\"\"Takes two numerical arrays of equal length\n",
    "    and counts the indices where the two are equal\"\"\"\n",
    "    return count_zero(array1 - array2)\n",
    "\n",
    "def evaluate_accuracy(training, test, k):\n",
    "    test_attributes = test.drop(columns=['Class'])\n",
    "    def classify_testrow(row):\n",
    "        return classify(training, row, k)\n",
    "    c = test_attributes.apply(classify_testrow, axis=1)\n",
    "    return count_equal(c, test['Class']) / len(test)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now for the grand reveal -- let's see how we did.  We'll arbitrarily use $k=5$."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9101123595505618"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_accuracy(training_set, test_set, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The accuracy rate isn't bad at all for a simple classifier."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Breast Cancer Diagnosis\n",
    "Now I want to do an example based on diagnosing breast cancer.  I was inspired by Brittany Wenger, who won the Google national science fair in 2012 a 17-year old high school student.  Here's Brittany:\n",
    "\n",
    "![Brittany Wenger](../../images/BrittanyWagner.jpg)\n",
    "\n",
    "Brittany's [science fair project](https://sites.google.com/a/googlesciencefair.com/science-fair-2012-project-64a91af142a459cfb486ed5cb05f803b2eb41354-1333130785-87/home) was to build a classification algorithm to diagnose breast cancer.  She won grand prize for building an algorithm whose accuracy was almost 99%. \n",
    "\n",
    "Let's see how well we can do, with the ideas we've learned in this course.\n",
    "\n",
    "So, let me tell you a little bit about the data set.  Basically, if a woman has a lump in her breast, the doctors may want to take a biopsy to see if it is cancerous.  There are several different procedures for doing that.  Brittany focused on fine needle aspiration (FNA), because it is less invasive than the alternatives.  The doctor gets a sample of the mass, puts it under a microscope, takes a picture, and a trained lab tech analyzes the picture to determine whether it is cancer or not.  We get a picture like one of the following:\n",
    "\n",
    "![benign](../../images/benign.png)\n",
    "\n",
    "![cancer](../../images/malignant.png)\n",
    "\n",
    "Unfortunately, distinguishing between benign vs malignant can be tricky.  So, researchers have studied the use of machine learning to help with this task.  The idea is that we'll ask the lab tech to analyze the image and compute various attributes: things like the typical size of a cell, how much variation there is among the cell sizes, and so on.  Then, we'll try to use this information to predict (classify) whether the sample is malignant or not.  We have a training set of past samples from women where the correct diagnosis is known, and we'll hope that our machine learning algorithm can use those to learn how to predict the diagnosis for future samples.\n",
    "\n",
    "We end up with the following data set.  For the \"Class\" column, 1 means malignant (cancer); 0 means benign (not cancer)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Clump Thickness</th>\n",
       "      <th>Uniformity of Cell Size</th>\n",
       "      <th>Uniformity of Cell Shape</th>\n",
       "      <th>Marginal Adhesion</th>\n",
       "      <th>Single Epithelial Cell Size</th>\n",
       "      <th>Bare Nuclei</th>\n",
       "      <th>Bland Chromatin</th>\n",
       "      <th>Normal Nucleoli</th>\n",
       "      <th>Mitoses</th>\n",
       "      <th>Class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>8</td>\n",
       "      <td>8</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>4</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>8</td>\n",
       "      <td>10</td>\n",
       "      <td>10</td>\n",
       "      <td>8</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>9</td>\n",
       "      <td>7</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>4</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Clump Thickness  Uniformity of Cell Size  Uniformity of Cell Shape  \\\n",
       "0                5                        1                         1   \n",
       "1                5                        4                         4   \n",
       "2                3                        1                         1   \n",
       "3                6                        8                         8   \n",
       "4                4                        1                         1   \n",
       "5                8                       10                        10   \n",
       "6                1                        1                         1   \n",
       "7                2                        1                         2   \n",
       "8                2                        1                         1   \n",
       "9                4                        2                         1   \n",
       "\n",
       "   Marginal Adhesion  Single Epithelial Cell Size  Bare Nuclei  \\\n",
       "0                  1                            2            1   \n",
       "1                  5                            7           10   \n",
       "2                  1                            2            2   \n",
       "3                  1                            3            4   \n",
       "4                  3                            2            1   \n",
       "5                  8                            7           10   \n",
       "6                  1                            2           10   \n",
       "7                  1                            2            1   \n",
       "8                  1                            2            1   \n",
       "9                  1                            2            1   \n",
       "\n",
       "   Bland Chromatin  Normal Nucleoli  Mitoses  Class  \n",
       "0                3                1        1      0  \n",
       "1                3                2        1      0  \n",
       "2                3                1        1      0  \n",
       "3                3                7        1      0  \n",
       "4                3                1        1      0  \n",
       "5                9                7        1      1  \n",
       "6                3                1        1      0  \n",
       "7                3                1        1      0  \n",
       "8                1                1        5      0  \n",
       "9                2                1        1      0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "patients = pd.read_csv(path_data + 'breast-cancer.csv').drop(columns=['ID'])\n",
    "patients.head(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So we have 9 different attributes.  I don't know how to make a 9-dimensional scatterplot of all of them, so I'm going to pick two and plot them:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Class</th>\n",
       "      <th>Clump Thickness</th>\n",
       "      <th>Uniformity of Cell Size</th>\n",
       "      <th>Uniformity of Cell Shape</th>\n",
       "      <th>Marginal Adhesion</th>\n",
       "      <th>Single Epithelial Cell Size</th>\n",
       "      <th>Bare Nuclei</th>\n",
       "      <th>Bland Chromatin</th>\n",
       "      <th>Normal Nucleoli</th>\n",
       "      <th>Mitoses</th>\n",
       "      <th>Color</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>4</td>\n",
       "      <td>4</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>10</td>\n",
       "      <td>3</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>gold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>gold</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Class  Clump Thickness  Uniformity of Cell Size  Uniformity of Cell Shape  \\\n",
       "0      0                5                        1                         1   \n",
       "1      0                5                        4                         4   \n",
       "2      0                3                        1                         1   \n",
       "\n",
       "   Marginal Adhesion  Single Epithelial Cell Size  Bare Nuclei  \\\n",
       "0                  1                            2            1   \n",
       "1                  5                            7           10   \n",
       "2                  1                            2            2   \n",
       "\n",
       "   Bland Chromatin  Normal Nucleoli  Mitoses Color  \n",
       "0                3                1        1  gold  \n",
       "1                3                2        1  gold  \n",
       "2                3                1        1  gold  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "color_table = pd.DataFrame(\n",
    "    {'Class':np.array([1, 0]),\n",
    "    'Color':np.array(['darkblue', 'gold'])}\n",
    ")\n",
    "patients_with_colors = pd.merge(patients, color_table, on='Class')\n",
    "\n",
    "patent_label = patients_with_colors.pop('Class')\n",
    "\n",
    "patients_with_colors.insert(0, 'Class', patent_label)\n",
    "\n",
    "patients_with_colors.head(3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 31056 characters]\n",
      "text/plain": [
       "<Figure size 504x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "pwc_darkblue = patients_with_colors[patients_with_colors['Color'] == 'darkblue']\n",
    "pwc_gold = patients_with_colors[patients_with_colors['Color'] == 'gold']\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(7,6))\n",
    "\n",
    "ax.scatter(pwc_darkblue['Bland Chromatin'], \n",
    "           pwc_darkblue['Single Epithelial Cell Size'],  \n",
    "           label='Color=darkblue', \n",
    "           color='darkblue')\n",
    "\n",
    "ax.scatter(pwc_gold['Bland Chromatin'], \n",
    "           pwc_gold['Single Epithelial Cell Size'],  \n",
    "           label='Color=gold', \n",
    "           color='gold')\n",
    "\n",
    "x_label = 'Bland Chromatin'\n",
    "\n",
    "y_label = 'Single Epithelial Cell Size'\n",
    "\n",
    "plt.ylabel(y_label)\n",
    "\n",
    "plt.xlabel(x_label)\n",
    "\n",
    "ax.legend(bbox_to_anchor=(1.04,1), loc=\"upper left\")\n",
    "\n",
    "plt.xlim(0, 11)\n",
    "plt.ylim(0, 11);\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Oops.  That plot is utterly misleading, because there are a bunch of points that have identical values for both the x- and y-coordinates.  To make it easier to see all the data points, I'm going to add a little bit of random jitter to the x- and y-values.  Here's how that looks:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "tags": [
     "remove_input"
    ]
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Bland Chromatin (jittered)</th>\n",
       "      <th>Single Epithelial Cell Size (jittered)</th>\n",
       "      <th>Class</th>\n",
       "      <th>Color</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3.083564</td>\n",
       "      <td>1.974698</td>\n",
       "      <td>0</td>\n",
       "      <td>gold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>3.018529</td>\n",
       "      <td>7.027071</td>\n",
       "      <td>0</td>\n",
       "      <td>gold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2.986660</td>\n",
       "      <td>1.905804</td>\n",
       "      <td>0</td>\n",
       "      <td>gold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3.093251</td>\n",
       "      <td>3.062657</td>\n",
       "      <td>0</td>\n",
       "      <td>gold</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3.055900</td>\n",
       "      <td>2.125152</td>\n",
       "      <td>0</td>\n",
       "      <td>gold</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Bland Chromatin (jittered)  Single Epithelial Cell Size (jittered)  Class  \\\n",
       "0                    3.083564                                1.974698      0   \n",
       "1                    3.018529                                7.027071      0   \n",
       "2                    2.986660                                1.905804      0   \n",
       "3                    3.093251                                3.062657      0   \n",
       "4                    3.055900                                2.125152      0   \n",
       "\n",
       "  Color  \n",
       "0  gold  \n",
       "1  gold  \n",
       "2  gold  \n",
       "3  gold  \n",
       "4  gold  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def randomize_column(a):\n",
    "    return a + np.random.normal(0.0, 0.09, size=len(a))\n",
    "\n",
    "jittered = pd.DataFrame(\n",
    "        {'Bland Chromatin (jittered)':randomize_column(patients['Bland Chromatin']),\n",
    "        'Single Epithelial Cell Size (jittered)':randomize_column(patients['Single Epithelial Cell Size']),\n",
    "        'Class':patients['Class']})\n",
    "\n",
    "\n",
    "jwc = pd.merge(jittered, color_table, on='Class')\n",
    "\n",
    "jwc.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 45580 characters]\n",
      "text/plain": [
       "<Figure size 504x432 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "jwc_darkblue = jwc[jwc['Color'] == 'darkblue']\n",
    "jwc_gold = jwc[jwc['Color'] == 'gold']\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(7,6))\n",
    "\n",
    "ax.scatter(jwc_darkblue['Bland Chromatin (jittered)'], \n",
    "           jwc_darkblue['Single Epithelial Cell Size (jittered)'],  \n",
    "           label='Color=darkblue', \n",
    "           color='darkblue')\n",
    "\n",
    "ax.scatter(jwc_gold['Bland Chromatin (jittered)'], \n",
    "           jwc_gold['Single Epithelial Cell Size (jittered)'],  \n",
    "           label='Color=gold', \n",
    "           color='gold')\n",
    "\n",
    "x_label = 'Bland Chromatin (jittered)'\n",
    "\n",
    "y_label = 'Single Epithelial Cell Size (jittered)'\n",
    "\n",
    "plt.ylabel(y_label)\n",
    "\n",
    "plt.xlabel(x_label)\n",
    "\n",
    "ax.legend(bbox_to_anchor=(1.04,1), loc=\"upper left\")\n",
    "\n",
    "plt.xlim(0, 11)\n",
    "plt.ylim(0, 11);\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For instance, you can see there are lots of samples with chromatin = 2 and epithelial cell size = 2; all non-cancerous.\n",
    "\n",
    "Keep in mind that the jittering is just for visualization purposes, to make it easier to get a feeling for the data.  We're ready to work with the data now, and we'll use the original (unjittered) data."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First we'll create a training set and a test set. The data set has 683 patients, so we'll randomly permute the data set and put 342 of them in the training set and the remaining 341 in the test set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "shuffled_patients = patients.sample(683, replace=False) \n",
    "training_set = shuffled_patients.take(np.arange(342))\n",
    "test_set  = shuffled_patients.take(np.arange(342, 683))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's stick with 5 nearest neighbors, and see how well our classifier does."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9706744868035191"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluate_accuracy(training_set, test_set, 5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Over 96% accuracy.  Not bad!  Once again, pretty darn good for such a simple technique.\n",
    "\n",
    "As a footnote, you might have noticed that Brittany Wenger did even better.  What techniques did she use? One key innovation is that she incorporated a confidence score into her results: her algorithm had a way to determine when it was not able to make a confident prediction, and for those patients, it didn't even try to predict their diagnosis.  Her algorithm was 99% accurate on the patients where it made a prediction -- so that extension seemed to help quite a bit."
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
