Category Histogram Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn import preprocessing\n",
    "from scipy.spatial import distance\n",
    "import matplotlib.pyplot as plt\n",
    "np.random.seed(42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "dfx = pd.read_csv(\"PLIC-milano-processed.csv\", sep=';')\n",
    "dfx = dfx.select_dtypes(exclude=['object', 'float64', 'float32', 'float'])\n",
    "dfx = dfx.drop(labels = ['Unnamed: 0', 'Unnamed: 0.1', 'cod pcs'], axis=1) #\n",
    "dfx = dfx.dropna(how='any', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abdomen</th>\n",
       "      <th>angina</th>\n",
       "      <th>remote angina</th>\n",
       "      <th>antiaggregators</th>\n",
       "      <th>antidiabetics</th>\n",
       "      <th>antihypertensives</th>\n",
       "      <th>respiratory system</th>\n",
       "      <th>arrhythmia</th>\n",
       "      <th>major arrhythmias</th>\n",
       "      <th>arteriopathyortic</th>\n",
       "      <th>...</th>\n",
       "      <th>remote decompensation</th>\n",
       "      <th>gender</th>\n",
       "      <th>NS blows</th>\n",
       "      <th>statins</th>\n",
       "      <th>steatosis</th>\n",
       "      <th>TIA</th>\n",
       "      <th>Remote TIA</th>\n",
       "      <th>thyroidpetto</th>\n",
       "      <th>thyroid nodules</th>\n",
       "      <th>venous thrombosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "      <td>5780.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.010035</td>\n",
       "      <td>0.007612</td>\n",
       "      <td>0.024221</td>\n",
       "      <td>0.135813</td>\n",
       "      <td>0.037543</td>\n",
       "      <td>0.359170</td>\n",
       "      <td>0.012111</td>\n",
       "      <td>-0.244810</td>\n",
       "      <td>-0.748270</td>\n",
       "      <td>0.007093</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003633</td>\n",
       "      <td>0.589619</td>\n",
       "      <td>-0.653114</td>\n",
       "      <td>0.220242</td>\n",
       "      <td>0.017474</td>\n",
       "      <td>0.009862</td>\n",
       "      <td>0.020761</td>\n",
       "      <td>0.086159</td>\n",
       "      <td>-0.005017</td>\n",
       "      <td>0.013495</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.112713</td>\n",
       "      <td>0.086924</td>\n",
       "      <td>0.153749</td>\n",
       "      <td>0.342620</td>\n",
       "      <td>0.190105</td>\n",
       "      <td>0.479799</td>\n",
       "      <td>0.118502</td>\n",
       "      <td>0.441919</td>\n",
       "      <td>0.438013</td>\n",
       "      <td>0.083930</td>\n",
       "      <td>...</td>\n",
       "      <td>0.060172</td>\n",
       "      <td>0.491945</td>\n",
       "      <td>0.653963</td>\n",
       "      <td>0.414446</td>\n",
       "      <td>0.131041</td>\n",
       "      <td>0.098823</td>\n",
       "      <td>0.142596</td>\n",
       "      <td>0.363381</td>\n",
       "      <td>0.210000</td>\n",
       "      <td>0.118352</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.250000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.750000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>8 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           abdomen       angina  remote angina  antiaggregators  \\\n",
       "count  5780.000000  5780.000000    5780.000000      5780.000000   \n",
       "mean      0.010035     0.007612       0.024221         0.135813   \n",
       "std       0.112713     0.086924       0.153749         0.342620   \n",
       "min      -1.000000     0.000000       0.000000         0.000000   \n",
       "25%       0.000000     0.000000       0.000000         0.000000   \n",
       "50%       0.000000     0.000000       0.000000         0.000000   \n",
       "75%       0.000000     0.000000       0.000000         0.000000   \n",
       "max       1.000000     1.000000       1.000000         1.000000   \n",
       "\n",
       "       antidiabetics  antihypertensives  respiratory system   arrhythmia  \\\n",
       "count    5780.000000        5780.000000         5780.000000  5780.000000   \n",
       "mean        0.037543           0.359170            0.012111    -0.244810   \n",
       "std         0.190105           0.479799            0.118502     0.441919   \n",
       "min         0.000000           0.000000           -1.000000    -1.000000   \n",
       "25%         0.000000           0.000000            0.000000    -0.250000   \n",
       "50%         0.000000           0.000000            0.000000     0.000000   \n",
       "75%         0.000000           1.000000            0.000000     0.000000   \n",
       "max         1.000000           1.000000            1.000000     1.000000   \n",
       "\n",
       "       major arrhythmias  arteriopathyortic  ...  remote decompensation  \\\n",
       "count        5780.000000        5780.000000  ...            5780.000000   \n",
       "mean           -0.748270           0.007093  ...               0.003633   \n",
       "std             0.438013           0.083930  ...               0.060172   \n",
       "min            -1.000000           0.000000  ...               0.000000   \n",
       "25%            -1.000000           0.000000  ...               0.000000   \n",
       "50%            -1.000000           0.000000  ...               0.000000   \n",
       "75%            -0.750000           0.000000  ...               0.000000   \n",
       "max             1.000000           1.000000  ...               1.000000   \n",
       "\n",
       "            gender     NS blows      statins    steatosis          TIA  \\\n",
       "count  5780.000000  5780.000000  5780.000000  5780.000000  5780.000000   \n",
       "mean      0.589619    -0.653114     0.220242     0.017474     0.009862   \n",
       "std       0.491945     0.653963     0.414446     0.131041     0.098823   \n",
       "min       0.000000    -1.000000     0.000000     0.000000     0.000000   \n",
       "25%       0.000000    -1.000000     0.000000     0.000000     0.000000   \n",
       "50%       1.000000    -1.000000     0.000000     0.000000     0.000000   \n",
       "75%       1.000000    -1.000000     0.000000     0.000000     0.000000   \n",
       "max       1.000000     1.000000     1.000000     1.000000     1.000000   \n",
       "\n",
       "        Remote TIA  thyroidpetto  thyroid nodules  venous thrombosis  \n",
       "count  5780.000000   5780.000000      5780.000000        5780.000000  \n",
       "mean      0.020761      0.086159        -0.005017           0.013495  \n",
       "std       0.142596      0.363381         0.210000           0.118352  \n",
       "min       0.000000     -1.000000        -1.000000          -1.000000  \n",
       "25%       0.000000      0.000000         0.000000           0.000000  \n",
       "50%       0.000000      0.000000         0.000000           0.000000  \n",
       "75%       0.000000      0.000000         0.000000           0.000000  \n",
       "max       1.000000      1.000000         1.000000           1.000000  \n",
       "\n",
       "[8 rows x 82 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfx = dfx[dfx.columns[dfx.max()<=1]]\n",
    "dfx.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5780, 82)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = dfx.values\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[ 174414  539115  921593 1191727 1317950 1289221 1156797  963258  775712\n",
      "  606548  463719  349076  281732  291911  391832  563150  712149  799503\n",
      "  803702  743001  636866  515805  390344  281986  194152  128659   80686\n",
      "   49933   31066   19844   13222    7916    5506    3839    2471    1492\n",
      "     786     337     150      69      38      19       7       2       5]\n",
      "44.0\n",
      "0.0\n"
     ]
    }
   ],
   "source": [
    "hamming = distance.pdist(X, 'hamming')*82\n",
    "#np.histogram(hamming, np.arange(83))\n",
    "freq = np.bincount(hamming.astype(int))\n",
    "print(freq)\n",
    "print(max(hamming))\n",
    "print(min(hamming))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEICAYAAABxiqLiAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjAsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+17YcXAAAVZklEQVR4nO3df/BddX3n8efLIJRqLb+Cg0lqsGaslN1GzQJb7NZKF4M4G5yBCrOVlGGbjgNT3XGnRv7Bau3izCpbZpRZlNTQWoGCDtmKpSky025XKV+U5acOEVPzNRGCCYi1wgbf+8f9fNebcL+/PoH7TczzMXPnnvs+n3M+5545yet7Pufce1NVSJI0Xy9a6A2QJB2cDBBJUhcDRJLUxQCRJHUxQCRJXQwQSVIXA0QakmRrkt8cUf+1JN9YiG2SDlQGiDQHVfX3VfWa2dol+UCSPx/HNkkLzQCRDhJJDlvobZCGGSDSc61Mcm+SJ5PckORnkrwpyeRUgyTvS/KdJE8l+UaSM5KsBi4D3pHkB0n+T2v7iiSbkuxKsiXJ7w6t58gkG5PsTvJQkj/Yp5+tra97gX9OcliS9Um+2fp+MMnbh9r/TpJ/SHJlkieSPJLkV1t9W5LHkqwdy17UTz3/opGe67eA1cCPgH8Afgf4+tTMJK8BLgX+TVVtT7IcWFRV30zyx8Crq+q3h9b3WeAB4BXALwGbkzxSVbcDlwPLgVcBLwFuHbE9FwBnA49X1Z4k3wR+DfgucB7w50leXVU7WvtTgU8BxwJ/CFwP/E/g1cCvAzcnubmqftC9hyQ8A5FGuaqqtlfVLgb/8a7cZ/6zwBHASUleXFVbq+qbo1aUZBnwRuB9VfWjqrqHwX/u72xNfgv446raXVWTwFXTbM+2qvoXgKr6y7Z9P66qG4CHgVOG2n+rqv60qp4FbgCWAR+sqqer6m+AZxiEibRfDBDpub47NP1D4KXDM6tqC/Ae4APAY0muT/KKadb1CmBXVT01VPsnYMnQ/G1D84anR9aSXJjknjZE9QRwMnDcUJNHh6anQmff2l7vSephgEgdquovquqNwCuBAj4yNWufptuBY5L83FDtF4DvtOkdwNKhectGdTc1keSVwCcZDKEdW1VHAfcD6XwrUjcDRJqnJK9J8uYkRzC4TvIvDIa1YPDX//IkLwKoqm3A/wb+a7sY/6+Bi4HPtPY3Au9PcnSSJQyCYSYvYRAoO9u2XMTgDEQaOwNEmr8jgCuAxxkMdx3P4O4rgL9sz99L8tU2fQGDC+Xbgc8Dl1fV5jbvg8Ak8C3gb4GbgKen67iqHgQ+CnyZQVj9KwYX+qWxiz8oJR04krwLOL+qfn2ht0WajWcg0gJKckKS05O8qN0e/F4GZynSAc/PgUgL63DgfwAnAk8w+MzGJxZ0i6Q5cghLktTFISxJUpdDZgjruOOOq+XLly/0ZkjSQeXuu+9+vKoWj5p3yATI8uXLmZiYWOjNkKSDSpJ/mm6eQ1iSpC4GiCSpiwEiSepigEiSuhggkqQuBogkqYsBIknqYoBIkroYIJKkLofMJ9EXyvL1X5hx/tYrzh7TlkjS88szEElSFwNEktTFAJEkdTFAJEldZg2QJMuS3JHkoSQPJHl3q38gyXeS3NMebx1a5v1JtiT5RpK3DNVXt9qWJOuH6icmuTPJw0luSHJ4qx/RXm9p85fP1ockaTzmcgayB3hvVb0WOA24JMlJbd6VVbWyPW4FaPPOB34ZWA18IsmiJIuAjwNnAScBFwyt5yNtXSuA3cDFrX4xsLuqXg1c2dpN20f3XpAkzdusAVJVO6rqq236KeAhYMkMi6wBrq+qp6vqW8AW4JT22FJVj1TVM8D1wJokAd4M3NSW3wicM7SujW36JuCM1n66PiRJYzKvayBtCOl1wJ2tdGmSe5NsSHJ0qy0Btg0tNtlq09WPBZ6oqj371PdaV5v/ZGs/3br23d51SSaSTOzcuXM+b1WSNIs5B0iSlwI3A++pqu8DVwO/CKwEdgAfnWo6YvHqqPesa+9C1TVVtaqqVi1ePPInfSVJneYUIElezCA8PlNVnwOoqker6tmq+jHwSX4yhDQJLBtafCmwfYb648BRSQ7bp77Xutr8nwd2zbAuSdKYzOUurADXAg9V1ceG6icMNXs7cH+b3gSc3+6gOhFYAfwjcBewot1xdTiDi+CbqqqAO4Bz2/JrgVuG1rW2TZ8LfKm1n64PSdKYzOW7sE4H3gncl+SeVruMwV1UKxkMHW0Ffg+gqh5IciPwIIM7uC6pqmcBklwK3AYsAjZU1QNtfe8Drk/yR8DXGAQW7fnPkmxhcOZx/mx9SJLGI4M/6H/6rVq1qiYmJsber1+mKOlgluTuqlo1ap6fRJckdTFAJEldDBBJUhcDRJLUxQCRJHUxQCRJXQwQSVIXA0SS1MUAkSR1MUAkSV0MEElSFwNEktTFAJEkdTFAJEldDBBJUhcDRJLUxQCRJHUxQCRJXQwQSVIXA0SS1MUAkSR1MUAkSV0MEElSFwNEktTlsIXegEPd8vVfmHbe1ivOHuOWSNL8GCDPg5lCQJJ+WjmEJUnqYoBIkroYIJKkLgaIJKmLASJJ6jJrgCRZluSOJA8leSDJu1v9mCSbkzzcno9u9SS5KsmWJPcmef3Quta29g8nWTtUf0OS+9oyVyVJbx+SpPGYyxnIHuC9VfVa4DTgkiQnAeuB26tqBXB7ew1wFrCiPdYBV8MgDIDLgVOBU4DLpwKhtVk3tNzqVp9XH5Kk8Zk1QKpqR1V9tU0/BTwELAHWABtbs43AOW16DXBdDXwFOCrJCcBbgM1VtauqdgObgdVt3suq6stVVcB1+6xrPn1IksZkXtdAkiwHXgfcCby8qnbAIGSA41uzJcC2ocUmW22m+uSIOh197Lu965JMJJnYuXPnfN6qJGkWcw6QJC8FbgbeU1Xfn6npiFp11GfcnLksU1XXVNWqqlq1ePHiWVYpSZqPOQVIkhczCI/PVNXnWvnRqWGj9vxYq08Cy4YWXwpsn6W+dES9pw9J0pjM5S6sANcCD1XVx4ZmbQKm7qRaC9wyVL+w3Sl1GvBkG366DTgzydHt4vmZwG1t3lNJTmt9XbjPuubThyRpTObyZYqnA+8E7ktyT6tdBlwB3JjkYuDbwHlt3q3AW4EtwA+BiwCqaleSDwF3tXYfrKpdbfpdwKeBI4Evtgfz7UOSND6zBkhV/S9GX3MAOGNE+wIumWZdG4ANI+oTwMkj6t+bbx+SpPHwk+iSpC4GiCSpiwEiSepigEiSuhggkqQuBogkqYsBIknqYoBIkroYIJKkLgaIJKmLASJJ6mKASJK6GCCSpC4GiCSpiwEiSepigEiSuhggkqQuBogkqYsBIknqYoBIkroYIJKkLgaIJKmLASJJ6mKASJK6GCCSpC4GiCSpiwEiSepigEiSuhggkqQuBogkqcthszVIsgF4G/BYVZ3cah8AfhfY2ZpdVlW3tnnvBy4GngV+v6pua/XVwJ8Ai4BPVdUVrX4icD1wDPBV4J1V9UySI4DrgDcA3wPeUVVbZ+pDL7zl678w4/ytV5w9pi2RtNDmcgbyaWD1iPqVVbWyPabC4yTgfOCX2zKfSLIoySLg48BZwEnABa0twEfaulYAuxkEA+15d1W9GriytZu2j/m9bUnS/po1QKrq74Bdc1zfGuD6qnq6qr4FbAFOaY8tVfVIVT3D4IxjTZIAbwZuastvBM4ZWtfGNn0TcEZrP10fkqQxmnUIawaXJrkQmADeW1W7gSXAV4baTLYawLZ96qcCxwJPVNWeEe2XTC1TVXuSPNnaz9SHngezDVNJEvRfRL8a+EVgJbAD+GirZ0Tb6qj3rOs5kqxLMpFkYufOnaOaSJI6dQVIVT1aVc9W1Y+BT/KTIaRJYNlQ06XA9hnqjwNHJTlsn/pe62rzf57BUNp06xq1nddU1aqqWrV48eKetypJmkZXgCQ5Yejl24H72/Qm4PwkR7S7q1YA/wjcBaxIcmKSwxlcBN9UVQXcAZzbll8L3DK0rrVt+lzgS639dH1IksZoLrfxfhZ4E3BckkngcuBNSVYyGDraCvweQFU9kORG4EFgD3BJVT3b1nMpcBuD23g3VNUDrYv3Adcn+SPga8C1rX4t8GdJtjA48zh/tj4kSeMza4BU1QUjyteOqE21/zDw4RH1W4FbR9QfYcRdVFX1I+C8+fQhSRofP4kuSepigEiSuhggkqQuBogkqcv+fBJdLzC/uFDSgcwzEElSFwNEktTFAJEkdTFAJEldDBBJUhcDRJLUxQCRJHUxQCRJXQwQSVIXA0SS1MUAkSR1MUAkSV0MEElSFwNEktTFAJEkdTFAJEldDBBJUhd/kVDPq5l+RdFfUJR+ungGIknqYoBIkroYIJKkLgaIJKmLASJJ6mKASJK6GCCSpC4GiCSpy6wBkmRDkseS3D9UOybJ5iQPt+ejWz1JrkqyJcm9SV4/tMza1v7hJGuH6m9Icl9b5qok6e1DkjQ+czkD+TSwep/aeuD2qloB3N5eA5wFrGiPdcDVMAgD4HLgVOAU4PKpQGht1g0tt7qnD0nSeM0aIFX1d8CufcprgI1teiNwzlD9uhr4CnBUkhOAtwCbq2pXVe0GNgOr27yXVdWXq6qA6/ZZ13z6kCSNUe81kJdX1Q6A9nx8qy8Btg21m2y1meqTI+o9fTxHknVJJpJM7Ny5c15vUJI0s+f7InpG1Kqj3tPHc4tV11TVqqpatXjx4llWK0maj94AeXRq2Kg9P9bqk8CyoXZLge2z1JeOqPf0IUkao94A2QRM3Um1FrhlqH5hu1PqNODJNvx0G3BmkqPbxfMzgdvavKeSnNbuvrpwn3XNpw9J0hjN+nsgST4LvAk4Lskkg7uprgBuTHIx8G3gvNb8VuCtwBbgh8BFAFW1K8mHgLtauw9W1dSF+XcxuNPrSOCL7cF8+9DczfSbHZI0V7MGSFVdMM2sM0a0LeCSadazAdgwoj4BnDyi/r359iFJGh8/iS5J6mKASJK6GCCSpC4GiCSpiwEiSepigEiSuhggkqQuBogkqYsBIknqMusn0XXgmukrSbZecfYYt0TSocgzEElSFwNEktTFAJEkdTFAJEldDBBJUhcDRJLUxQCRJHXxcyBz4E/AStJzeQYiSepigEiSujiEpbGZbSjQr1+RDi6egUiSuhggkqQuBogkqYsBIknqYoBIkroYIJKkLgaIJKmLnwP5KeXXr0h6oXkGIknqYoBIkrrsV4Ak2ZrkviT3JJlotWOSbE7ycHs+utWT5KokW5Lcm+T1Q+tZ29o/nGTtUP0Nbf1b2rKZqQ9J0vg8H2cgv1FVK6tqVXu9Hri9qlYAt7fXAGcBK9pjHXA1DMIAuBw4FTgFuHwoEK5ubaeWWz1LH5KkMXkhhrDWABvb9EbgnKH6dTXwFeCoJCcAbwE2V9WuqtoNbAZWt3kvq6ovV1UB1+2zrlF9SJLGZH8DpIC/SXJ3knWt9vKq2gHQno9v9SXAtqFlJ1ttpvrkiPpMfewlybokE0kmdu7c2fkWJUmj7O9tvKdX1fYkxwObk3x9hrYZUauO+pxV1TXANQCrVq2a17KSpJnt1xlIVW1vz48Bn2dwDePRNvxEe36sNZ8Elg0tvhTYPkt96Yg6M/QhSRqT7gBJ8pIkPzc1DZwJ3A9sAqbupFoL3NKmNwEXtruxTgOebMNPtwFnJjm6XTw/E7itzXsqyWnt7qsL91nXqD4kSWOyP0NYLwc+3+6sPQz4i6r66yR3ATcmuRj4NnBea38r8FZgC/BD4CKAqtqV5EPAXa3dB6tqV5t+F/Bp4Ejgi+0BcMU0fUiSxqQ7QKrqEeBXRtS/B5wxol7AJdOsawOwYUR9Ajh5rn1IksbHT6JLkroYIJKkLgaIJKmLASJJ6mKASJK6GCCSpC4GiCSpiwEiSerib6LrgDHT77hvveLsMW6JpLnwDESS1MUAkSR1MUAkSV0MEElSFwNEktTFAJEkdTFAJEldDBBJUhcDRJLUxQCRJHUxQCRJXQwQSVIXA0SS1MUAkSR18evcdVDwq96lA49nIJKkLgaIJKmLASJJ6mKASJK6GCCSpC4GiCSpi7fx6qA30y2+4G2+0gvloD4DSbI6yTeSbEmyfqG3R5IOJQftGUiSRcDHgX8PTAJ3JdlUVQ8u7JbpQOOHEKUXxkEbIMApwJaqegQgyfXAGsAA0ZzNNvw1E8NHh7qDOUCWANuGXk8Cpw43SLIOWNde/iDJNzr7Og54vHPZQ80hs6/ykf1a/JDZT/vJ/TQ3L+R+euV0Mw7mAMmIWu31ouoa4Jr97iiZqKpV+7ueQ4H7am7cT3PjfpqbhdpPB/NF9Elg2dDrpcD2BdoWSTrkHMwBchewIsmJSQ4Hzgc2LfA2SdIh46AdwqqqPUkuBW4DFgEbquqBF6i7/R4GO4S4r+bG/TQ37qe5WZD9lKqavZUkSfs4mIewJEkLyACRJHUxQGbh16WMlmRZkjuSPJTkgSTvbvVjkmxO8nB7Pnqht/VAkGRRkq8l+av2+sQkd7b9dEO7EeSQl+SoJDcl+Xo7tv6tx9RzJfnP7d/d/Uk+m+RnFuKYMkBmMPR1KWcBJwEXJDlpYbfqgLEHeG9VvRY4Dbik7Zv1wO1VtQK4vb0WvBt4aOj1R4Ar237aDVy8IFt14PkT4K+r6peAX2GwzzymhiRZAvw+sKqqTmZwE9H5LMAxZYDM7P9/XUpVPQNMfV3KIa+qdlTVV9v0Uwz+oS9hsH82tmYbgXMWZgsPHEmWAmcDn2qvA7wZuKk1cT8BSV4G/DvgWoCqeqaqnsBjapTDgCOTHAb8LLCDBTimDJCZjfq6lCULtC0HrCTLgdcBdwIvr6odMAgZ4PiF27IDxn8H/gD4cXt9LPBEVe1prz2uBl4F7AT+tA33fSrJS/CY2ktVfQf4b8C3GQTHk8DdLMAxZYDMbNavSznUJXkpcDPwnqr6/kJvz4EmyduAx6rq7uHyiKYeV4O/ql8PXF1VrwP+mUN8uGqUdg1oDXAi8ArgJQyG2ff1gh9TBsjM/LqUGSR5MYPw+ExVfa6VH01yQpt/AvDYQm3fAeJ04D8k2cpgCPTNDM5IjmrDD+BxNWUSmKyqO9vrmxgEisfU3n4T+FZV7ayq/wt8DvhVFuCYMkBm5telTKON418LPFRVHxuatQlY26bXAreMe9sOJFX1/qpaWlXLGRw/X6qq/wjcAZzbmh3y+wmgqr4LbEvymlY6g8HPM3hM7e3bwGlJfrb9O5zaT2M/pvwk+iySvJXBX4xTX5fy4QXepANCkjcCfw/cx0/G9i9jcB3kRuAXGBzo51XVrgXZyANMkjcB/6Wq3pbkVQzOSI4Bvgb8dlU9vZDbdyBIspLBzQaHA48AFzH4Q9djakiSPwTeweBuyK8B/4nBNY+xHlMGiCSpi0NYkqQuBogkqYsBIknqYoBIkroYIJKkLgaIJKmLASJJ6vL/AGM2OM4hXCBrAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(hamming, bins = np.arange(0, 84, 2)) \n",
    "plt.title(\"histogram\") \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0,  1, -1])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a = dfx[\"respiratory system\"]\n",
    "a.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0, 0, 0, ..., 0, 0, 0]), Int64Index([0, 1, -1], dtype='int64'))"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.factorize(a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([2119, 2575, 3527, 3615, 4027, 4347]),)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.where(a==1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a[2048]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.factorize(a)[0][2047]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>abdomen</th>\n",
       "      <th>angina</th>\n",
       "      <th>remote angina</th>\n",
       "      <th>antiaggregators</th>\n",
       "      <th>antidiabetics</th>\n",
       "      <th>antihypertensives</th>\n",
       "      <th>respiratory system</th>\n",
       "      <th>arrhythmia</th>\n",
       "      <th>major arrhythmias</th>\n",
       "      <th>arteriopathyortic</th>\n",
       "      <th>...</th>\n",
       "      <th>remote decompensation</th>\n",
       "      <th>gender</th>\n",
       "      <th>NS blows</th>\n",
       "      <th>statins</th>\n",
       "      <th>steatosis</th>\n",
       "      <th>TIA</th>\n",
       "      <th>Remote TIA</th>\n",
       "      <th>thyroidpetto</th>\n",
       "      <th>thyroid nodules</th>\n",
       "      <th>venous thrombosis</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2 rows × 82 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   abdomen  angina  remote angina  antiaggregators  antidiabetics  \\\n",
       "0        0       0              0                0              0   \n",
       "1        0       0              0                0              0   \n",
       "\n",
       "   antihypertensives  respiratory system  arrhythmia  major arrhythmias  \\\n",
       "0                  0                   0           0                 -1   \n",
       "1                  0                   0           0                 -1   \n",
       "\n",
       "   arteriopathyortic  ...  remote decompensation  gender  NS blows  statins  \\\n",
       "0                  0  ...                      0       1        -1        0   \n",
       "1                  0  ...                      0       1        -1        0   \n",
       "\n",
       "   steatosis  TIA  Remote TIA  thyroidpetto  thyroid nodules  \\\n",
       "0          0    0           0             0                0   \n",
       "1          0    0           0             0                0   \n",
       "\n",
       "   venous thrombosis  \n",
       "0                  0  \n",
       "1                  0  \n",
       "\n",
       "[2 rows x 82 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfx.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'00000000-1000-10-10-10000000000000000-1-1-1001101000000000-10-1-1-1-1-1-1000-1-1-1101000001-10000000'"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "str(dfx.loc[0].values)[1:-1].replace(\"\\n\", \"\").replace(\" \", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xcat = dfx.apply(lambda r: str(r.values)[1:-1].replace(\"\\n\", \"\").replace(\" \", \"\"), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2773,)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.unique(pd.factorize(Xcat)[0]).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.0843851424450053"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "5780/2773"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    00000000-1000-10-10-10000000000000000-1-1-1001...\n",
       "1    00000000-1000-10-10-10000000000000000-1-1-1001...\n",
       "2    00000000-1000-10-10-10000000000000000-1-1-1001...\n",
       "3    0000000-1000-10010-100000000000000000-1-100100...\n",
       "4    10000000-1000-10-10-10000000000000000-1-1-1001...\n",
       "dtype: object"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Xcat.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "hashed = Xcat.apply(lambda s: hash(s))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([   0,    0,    1, ...,   19,  912, 2772]),\n",
       " Int64Index([-6437453522594871643, -1973562797831179909,  7989383210362685393,\n",
       "              1897610268733857411, -6374182330202693638, -4735450334074731900,\n",
       "              1791067613435069702,  8357015345315169091, -4209862562261959157,\n",
       "             -7697808878238306714,\n",
       "             ...\n",
       "              6036823350294092628,  1726643015932370530,  -384495652281676411,\n",
       "             -1668255873303378851,  8192179585000999648,  5820024989429433229,\n",
       "              4945168727032412646, -5754176173733311777, -2237646609820704005,\n",
       "              -202678240541446479],\n",
       "            dtype='int64', length=2773))"
      ]
     },
     "execution_count": 56,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.factorize(hashed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(array([0]), array(['asdas'], dtype=object))"
      ]
     },
     "execution_count": 59,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.factorize([\"asdas\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "---\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5780, 82)"
      ]
     },
     "execution_count": 64,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dfx.values.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "Xcat_arr = dfx.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "H_cat = distance.pdist(Xcat_arr, 'hamming')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5780, 5780)"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "H_cat_dense = distance.squareform(H_cat)\n",
    "H_cat_dense.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " 5\n",
      "0: (1054,)\n",
      "1: (1475,)\n",
      "2: (2410,)\n",
      "3: (450,)\n",
      "4: (391,)\n"
     ]
    }
   ],
   "source": [
    "for c in (5,):\n",
    "    kmeans = KMeans(n_clusters=c, precompute_distances=False, algorithm='elkan', random_state=42)\n",
    "    kmeans.fit(H_cat_dense)\n",
    "\n",
    "    print(\"\\n\", c)\n",
    "    for i in range(c):\n",
    "        print(\"{}: {}\".format(i, np.where(kmeans.labels_ == i)[0].shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [],
   "source": [
    "kmeans.labels_[kmeans.labels_ == 4] = 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0: (1054,)\n",
      "1: (1475,)\n",
      "2: (2410,)\n",
      "3: (841,)\n"
     ]
    }
   ],
   "source": [
    "for i in range(4):\n",
    "        print(\"{}: {}\".format(i, np.where(kmeans.labels_ == i)[0].shape))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [],
   "source": [
    "cat_vars = kmeans.labels_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}