work_1.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Part 1\n",
    "\n",
    "General Overview:\n",
    "\n",
    "1. Understand the shape of the data\n",
    "2. Data Exploration\n",
    "3. Data Cleaning\n",
    "4. Data Preprocessing for Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import re\n",
    "import calendar\n",
    "\n",
    "import seaborn as sns\n",
    "import matplotlib.style as style\n",
    "\n",
    "\n",
    "import matplotlib as mpl\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.cm as cm\n",
    "%matplotlib inline\n",
    "\n",
    "# grid: rgb(240,240,240)\n",
    "# background: rgb(200,200,200)\n",
    "style.use('fivethirtyeight')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#nlp\n",
    "import string\n",
    "import re    #for regex\n",
    "import nltk\n",
    "from nltk.corpus import stopwords # remove useless words or words that do not add value\n",
    "from nltk.stem.lancaster import LancasterStemmer # Convert words to the infinitive form - very aggressive transformation.\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n",
    "import spacy\n",
    "from nltk import pos_tag\n",
    "from nltk.stem.wordnet import WordNetLemmatizer \n",
    "from nltk.tokenize import word_tokenize\n",
    "# Tweet tokenizer does not split at apostophes which is what we want\n",
    "from nltk.tokenize import TweetTokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Understand the shape of the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Missing values\n",
    "def missing_values_func(df):\n",
    "    \"\"\"\n",
    "    @author: Cristobal Zamorano Astudillo\n",
    "    \n",
    "    Personalize Missing Data Function\n",
    "    \n",
    "    Paramaters\n",
    "    ----------\n",
    "    df : DataFrame of interest\n",
    "    \n",
    "    Returns\n",
    "    -------\n",
    "    A string with counting all the features of the input DataFrame. If some values are missing, then \n",
    "    function will return a DataFrame with the following:\n",
    "    - Index as the features with the missing values\n",
    "    - A `Missing Values Count` Feature that tells the exact number of rows that has a misssing value in that feature index.\n",
    "    - A `% of Total Values` that tells how much are the missing values of that feature with respect to the other missing values.\n",
    "    \"\"\"\n",
    "    \n",
    "    # Total missing values\n",
    "    mis_val = df.isnull().sum()\n",
    "\n",
    "    # Percentage of missing values\n",
    "    mis_val_percent = 100 * df.isnull().sum() / len(df)\n",
    "\n",
    "    # Make a table with the results\n",
    "    mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n",
    "\n",
    "    # Rename the columns\n",
    "    mis_val_table_ren_columns = mis_val_table.rename(\n",
    "        columns = {0 : 'Missing Values Count', 1 : '% of Total Values'})\n",
    "\n",
    "    # Sort the table by percentage of missing descending\n",
    "    mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
    "        mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
    "    '% of Total Values', ascending=False).round(1)\n",
    "\n",
    "\n",
    "    # Returns\n",
    "    if mis_val_table_ren_columns.empty:\n",
    "        return f'Your selected dataframe has  {df.shape[1]} features. There are  {mis_val_table_ren_columns.shape[0]} features that have missing values.'\n",
    "    else:\n",
    "        display(mis_val_table_ren_columns.style.background_gradient(cmap='Reds'))\n",
    "        return f'Your selected dataframe has  {df.shape[1]} features. There are  {mis_val_table_ren_columns.shape[0]} features that have missing values.'\n",
    "    \n",
    "def getting_to_know(df, question=None):\n",
    "    display(df.shape)\n",
    "    display(df.columns)\n",
    "    if question == 'y':\n",
    "        display(df.head())\n",
    "        display(df.tail())\n",
    "    print('--------------------------END--------------------------------')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/train.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Your selected dataframe has  8 features. There are  0 features that have missing values.'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "missing_values_func(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 159571 entries, 0 to 159570\n",
      "Data columns (total 8 columns):\n",
      "id               159571 non-null object\n",
      "comment_text     159571 non-null object\n",
      "toxic            159571 non-null int64\n",
      "severe_toxic     159571 non-null int64\n",
      "obscene          159571 non-null int64\n",
      "threat           159571 non-null int64\n",
      "insult           159571 non-null int64\n",
      "identity_hate    159571 non-null int64\n",
      "dtypes: int64(6), object(2)\n",
      "memory usage: 9.7+ MB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "So, above we can see a glimpse of what the data look like "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[\"Explanation\\nWhy the edits made under my username Hardcore Metallica Fan were reverted? They weren't vandalisms, just closure on some GAs after I voted at New York Dolls FAC. And please don't remove the template from the talk page since I'm retired now.89.205.38.27\",\n",
       " \"D'aww! He matches this background colour I'm seemingly stuck with. Thanks.  (talk) 21:51, January 11, 2016 (UTC)\",\n",
       " \"Hey man, I'm really not trying to edit war. It's just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page. He seems to care more about the formatting than the actual info.\"]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[train['comment_text'][i] for i in range(3)]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Data Exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(159571, 8)"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "Index(['id', 'comment_text', 'toxic', 'severe_toxic', 'obscene', 'threat',\n",
       "       'insult', 'identity_hate'],\n",
       "      dtype='object')"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>Explanation\\nWhy the edits made under my usern...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>D'aww! He matches this background colour I'm s...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>000113f07ec002fd</td>\n",
       "      <td>Hey man, I'm really not trying to edit war. It...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0001b41b1c6bb37e</td>\n",
       "      <td>\"\\nMore\\nI can't make any real suggestions on ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0001d958c54c6e35</td>\n",
       "      <td>You, sir, are my hero. Any chance you remember...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  0000997932d777bf  Explanation\\nWhy the edits made under my usern...      0   \n",
       "1  000103f0d9cfb60f  D'aww! He matches this background colour I'm s...      0   \n",
       "2  000113f07ec002fd  Hey man, I'm really not trying to edit war. It...      0   \n",
       "3  0001b41b1c6bb37e  \"\\nMore\\nI can't make any real suggestions on ...      0   \n",
       "4  0001d958c54c6e35  You, sir, are my hero. Any chance you remember...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  \n",
       "0             0        0       0       0              0  \n",
       "1             0        0       0       0              0  \n",
       "2             0        0       0       0              0  \n",
       "3             0        0       0       0              0  \n",
       "4             0        0       0       0              0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>159566</td>\n",
       "      <td>ffe987279560d7ff</td>\n",
       "      <td>\":::::And for the second time of asking, when ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159567</td>\n",
       "      <td>ffea4adeee384e90</td>\n",
       "      <td>You should be ashamed of yourself \\n\\nThat is ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159568</td>\n",
       "      <td>ffee36eab5c267c9</td>\n",
       "      <td>Spitzer \\n\\nUmm, theres no actual article for ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159569</td>\n",
       "      <td>fff125370e4aaaf3</td>\n",
       "      <td>And it looks like it was actually you who put ...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>159570</td>\n",
       "      <td>fff46fc426af1f9a</td>\n",
       "      <td>\"\\nAnd ... I really don't think you understand...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      id                                       comment_text  \\\n",
       "159566  ffe987279560d7ff  \":::::And for the second time of asking, when ...   \n",
       "159567  ffea4adeee384e90  You should be ashamed of yourself \\n\\nThat is ...   \n",
       "159568  ffee36eab5c267c9  Spitzer \\n\\nUmm, theres no actual article for ...   \n",
       "159569  fff125370e4aaaf3  And it looks like it was actually you who put ...   \n",
       "159570  fff46fc426af1f9a  \"\\nAnd ... I really don't think you understand...   \n",
       "\n",
       "        toxic  severe_toxic  obscene  threat  insult  identity_hate  \n",
       "159566      0             0        0       0       0              0  \n",
       "159567      0             0        0       0       0              0  \n",
       "159568      0             0        0       0       0              0  \n",
       "159569      0             0        0       0       0              0  \n",
       "159570      0             0        0       0       0              0  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--------------------------END--------------------------------\n"
     ]
    }
   ],
   "source": [
    "getting_to_know(train, 'y')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's see a quick rundown of the features counts and see how they relate to each other:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "toxic            15294\n",
       "severe_toxic      1595\n",
       "obscene           8449\n",
       "threat             478\n",
       "insult            7877\n",
       "identity_hate     1405\n",
       "dtype: int64"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.iloc[:, 2:].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.904156\n",
       "1    0.095844\n",
       "Name: toxic, dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['toxic'].value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.990004\n",
       "1    0.009996\n",
       "Name: severe_toxic, dtype: float64"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['severe_toxic'].value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.947052\n",
       "1    0.052948\n",
       "Name: obscene, dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['obscene'].value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.997004\n",
       "1    0.002996\n",
       "Name: threat, dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['threat'].value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.950636\n",
       "1    0.049364\n",
       "Name: insult, dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['insult'].value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    0.991195\n",
       "1    0.008805\n",
       "Name: identity_hate, dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['identity_hate'].value_counts(normalize = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_toxic_train = train[(train['toxic'] == 0)\n",
    "   & (train['severe_toxic'] == 0)\n",
    "   & (train['obscene'] == 0) \n",
    "   & (train['threat'] == 0) \n",
    "   & (train['insult'] == 0) \n",
    "   & (train['identity_hate'] == 0)]\n",
    "\n",
    "toxic_train = train[(train['toxic'] == 1)\n",
    "   | (train['severe_toxic'] == 1)\n",
    "   | (train['obscene'] == 1) \n",
    "   | (train['threat'] == 1) \n",
    "   | (train['insult'] == 1) \n",
    "   | (train['identity_hate'] == 1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train.shape[0] == (no_toxic_train.shape[0] + toxic_train.shape[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proportion of the No toxic comments : 0.8983211235124177\n"
     ]
    }
   ],
   "source": [
    "print(f'Proportion of the No toxic comments : {(no_toxic_train.shape[0] / train.shape[0])}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Proportion of the toxic comments : 0.10167887648758234\n"
     ]
    }
   ],
   "source": [
    "print(f'Proportion of the toxic comments : {(toxic_train.shape[0] / train.shape[0])}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Data is clearly very umbalanced. "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Graphs"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Data Cleaning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['i',\n",
       " 'me',\n",
       " 'my',\n",
       " 'myself',\n",
       " 'we',\n",
       " 'our',\n",
       " 'ours',\n",
       " 'ourselves',\n",
       " 'you',\n",
       " \"you're\",\n",
       " \"you've\",\n",
       " \"you'll\",\n",
       " \"you'd\",\n",
       " 'your',\n",
       " 'yours',\n",
       " 'yourself',\n",
       " 'yourselves',\n",
       " 'he',\n",
       " 'him',\n",
       " 'his',\n",
       " 'himself',\n",
       " 'she',\n",
       " \"she's\",\n",
       " 'her',\n",
       " 'hers',\n",
       " 'herself',\n",
       " 'it',\n",
       " \"it's\",\n",
       " 'its',\n",
       " 'itself',\n",
       " 'they',\n",
       " 'them',\n",
       " 'their',\n",
       " 'theirs',\n",
       " 'themselves',\n",
       " 'what',\n",
       " 'which',\n",
       " 'who',\n",
       " 'whom',\n",
       " 'this',\n",
       " 'that',\n",
       " \"that'll\",\n",
       " 'these',\n",
       " 'those',\n",
       " 'am',\n",
       " 'is',\n",
       " 'are',\n",
       " 'was',\n",
       " 'were',\n",
       " 'be',\n",
       " 'been',\n",
       " 'being',\n",
       " 'have',\n",
       " 'has',\n",
       " 'had',\n",
       " 'having',\n",
       " 'do',\n",
       " 'does',\n",
       " 'did',\n",
       " 'doing',\n",
       " 'a',\n",
       " 'an',\n",
       " 'the',\n",
       " 'and',\n",
       " 'but',\n",
       " 'if',\n",
       " 'or',\n",
       " 'because',\n",
       " 'as',\n",
       " 'until',\n",
       " 'while',\n",
       " 'of',\n",
       " 'at',\n",
       " 'by',\n",
       " 'for',\n",
       " 'with',\n",
       " 'about',\n",
       " 'against',\n",
       " 'between',\n",
       " 'into',\n",
       " 'through',\n",
       " 'during',\n",
       " 'before',\n",
       " 'after',\n",
       " 'above',\n",
       " 'below',\n",
       " 'to',\n",
       " 'from',\n",
       " 'up',\n",
       " 'down',\n",
       " 'in',\n",
       " 'out',\n",
       " 'on',\n",
       " 'off',\n",
       " 'over',\n",
       " 'under',\n",
       " 'again',\n",
       " 'further',\n",
       " 'then',\n",
       " 'once',\n",
       " 'here',\n",
       " 'there',\n",
       " 'when',\n",
       " 'where',\n",
       " 'why',\n",
       " 'how',\n",
       " 'all',\n",
       " 'any',\n",
       " 'both',\n",
       " 'each',\n",
       " 'few',\n",
       " 'more',\n",
       " 'most',\n",
       " 'other',\n",
       " 'some',\n",
       " 'such',\n",
       " 'no',\n",
       " 'nor',\n",
       " 'not',\n",
       " 'only',\n",
       " 'own',\n",
       " 'same',\n",
       " 'so',\n",
       " 'than',\n",
       " 'too',\n",
       " 'very',\n",
       " 's',\n",
       " 't',\n",
       " 'can',\n",
       " 'will',\n",
       " 'just',\n",
       " 'don',\n",
       " \"don't\",\n",
       " 'should',\n",
       " \"should've\",\n",
       " 'now',\n",
       " 'd',\n",
       " 'll',\n",
       " 'm',\n",
       " 'o',\n",
       " 're',\n",
       " 've',\n",
       " 'y',\n",
       " 'ain',\n",
       " 'aren',\n",
       " \"aren't\",\n",
       " 'couldn',\n",
       " \"couldn't\",\n",
       " 'didn',\n",
       " \"didn't\",\n",
       " 'doesn',\n",
       " \"doesn't\",\n",
       " 'hadn',\n",
       " \"hadn't\",\n",
       " 'hasn',\n",
       " \"hasn't\",\n",
       " 'haven',\n",
       " \"haven't\",\n",
       " 'isn',\n",
       " \"isn't\",\n",
       " 'ma',\n",
       " 'mightn',\n",
       " \"mightn't\",\n",
       " 'mustn',\n",
       " \"mustn't\",\n",
       " 'needn',\n",
       " \"needn't\",\n",
       " 'shan',\n",
       " \"shan't\",\n",
       " 'shouldn',\n",
       " \"shouldn't\",\n",
       " 'wasn',\n",
       " \"wasn't\",\n",
       " 'weren',\n",
       " \"weren't\",\n",
       " 'won',\n",
       " \"won't\",\n",
       " 'wouldn',\n",
       " \"wouldn't\"]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop = stopwords.words('english')\n",
    "stop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_data(df):\n",
    "\n",
    "#     # Lowercase letters\n",
    "#     df['comment_text'] = df['comment_text'].str.lower()\n",
    "    \n",
    "    # Remove special characters (i.e. @, $, %)\n",
    "    df['comment_text'] = df['comment_text'].str.replace(\"[^a-z0-9!@#\\$%\\^\\&\\*_\\-,\\.' ]\", \" \") \n",
    "    \n",
    "    # Remove punctuations - if you don't want to remove this, comment below out\n",
    "    df['comment_text'] = df['comment_text'].str.replace('[^\\w\\s]', \"\")\n",
    "    \n",
    "    # Remove random numbers\n",
    "    df['comment_text'] = df['comment_text'].str.replace(\"[^a-z' ]\", \"\")\n",
    "    \n",
    "    # Remove repeating characters and extra spaces \n",
    "    # Examples: 'whaaat' --> 'what' OR 'hello   bye' --> 'hello bye'\n",
    "    # Note: This does change words like \"look\" to \"lok\" cuz of the double letters, \n",
    "    # if we don't want this, we can remove the \\w in the regex pattern below.\n",
    "    repeat_pattern = re.compile(r'(\\w|\\s)\\1*')\n",
    "    match_substitution = r'\\1'\n",
    "    df['comment_text'] = df['comment_text'].apply(lambda x: repeat_pattern.sub(match_substitution, x))\n",
    "    \n",
    "    # Remove all numbers with letters\n",
    "    alphanumeric = lambda x: re.sub('\\w*\\d\\w*', ' ', x)\n",
    "\n",
    "    # '[%s]' % re.escape(string.punctuation),' ' - replace punctuation with white space\n",
    "    # .lower() - convert all strings to lowercase \n",
    "    punc_lower = lambda x: re.sub('[%s]' % re.escape(string.punctuation), ' ', x.lower())\n",
    "\n",
    "    # Remove all '\\n' in the string and replace it with a space\n",
    "    remove_n = lambda x: re.sub(\"\\n\", \" \", x)\n",
    "\n",
    "    # Remove all non-ascii characters \n",
    "    remove_non_ascii = lambda x: re.sub(r'[^\\x00-\\x7f]',r' ', x)\n",
    "\n",
    "\n",
    "    \n",
    "\n",
    "    # Apply all the lambda functions wrote previously through .map on the comments column\n",
    "    df['comment_text'] = df['comment_text'].map(alphanumeric).map(punc_lower).map(remove_n).map(remove_non_ascii)\n",
    "    \n",
    "    # Remove stop words\n",
    "#     pattern = re.compile(r'\\b('+stop_words_arr+r')\\b', flags=re.IGNORECASE).apply(str)\n",
    "#     remove_stop_words = lambda x: re.sub('\\s+', ' ', re.sub(pattern, '', x))\n",
    "#     pattern = re.compile(r'\\b(' + r'|'.join(stopwords.words('english')) + r')\\b\\s*')\n",
    "#     df['comment_text'] = pattern.sub('', str(df['comment_text']))\n",
    "    df['comment_text_clean']= df['comment_text'].apply(lambda x: ' '.join([word for word in x.split() if word not in (stop)]))\n",
    "\n",
    "    \n",
    "\n",
    "    \n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>comment_text</th>\n",
       "      <th>toxic</th>\n",
       "      <th>severe_toxic</th>\n",
       "      <th>obscene</th>\n",
       "      <th>threat</th>\n",
       "      <th>insult</th>\n",
       "      <th>identity_hate</th>\n",
       "      <th>comment_text_clean</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>0000997932d777bf</td>\n",
       "      <td>xplanation hy the edits made under my usernam...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>xplanation hy edits made username ardcore etal...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>000103f0d9cfb60f</td>\n",
       "      <td>aw e matches this background colour m semingl...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>aw e matches background colour semingly stuck ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>000113f07ec002fd</td>\n",
       "      <td>ey man m realy not trying to edit war ts just...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ey man realy trying edit war ts guy constantly...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0001b41b1c6bb37e</td>\n",
       "      <td>ore cant make any real sugestions on improvem...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ore cant make real sugestions improvement wond...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0001d958c54c6e35</td>\n",
       "      <td>ou sir are my hero ny chance you remember wha...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>ou sir hero ny chance remember page thats</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id                                       comment_text  toxic  \\\n",
       "0  0000997932d777bf   xplanation hy the edits made under my usernam...      0   \n",
       "1  000103f0d9cfb60f   aw e matches this background colour m semingl...      0   \n",
       "2  000113f07ec002fd   ey man m realy not trying to edit war ts just...      0   \n",
       "3  0001b41b1c6bb37e   ore cant make any real sugestions on improvem...      0   \n",
       "4  0001d958c54c6e35   ou sir are my hero ny chance you remember wha...      0   \n",
       "\n",
       "   severe_toxic  obscene  threat  insult  identity_hate  \\\n",
       "0             0        0       0       0              0   \n",
       "1             0        0       0       0              0   \n",
       "2             0        0       0       0              0   \n",
       "3             0        0       0       0              0   \n",
       "4             0        0       0       0              0   \n",
       "\n",
       "                                  comment_text_clean  \n",
       "0  xplanation hy edits made username ardcore etal...  \n",
       "1  aw e matches background colour semingly stuck ...  \n",
       "2  ey man realy trying edit war ts guy constantly...  \n",
       "3  ore cant make real sugestions improvement wond...  \n",
       "4          ou sir hero ny chance remember page thats  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = clean_data(train)\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[' xplanation hy the edits made under my username ardcore etalica an were reverted hey werent vandalisms just closure on some s after voted at ew ork ols nd please dont remove the template from the talk page since m retired now',\n",
       " ' aw e matches this background colour m semingly stuck with hanks talk anuary ',\n",
       " ' ey man m realy not trying to edit war ts just that this guy is constantly removing relevant information and talking to me through edits instead of my talk page e sems to care more about the formating than the actual info']"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[data['comment_text'][i] for i in range(3)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# data.to_csv('./data/output/cleaned_train_data.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 4. Data Preprocessing for Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "toxic_df = data.loc[:,['id', 'comment_text', 'toxic']]\n",
    "severe_toxic_df = data.loc[:,['id', 'comment_text', 'severe_toxic']]\n",
    "obscene_df = data.loc[:,['id', 'comment_text', 'obscene']]\n",
    "threat_df = data.loc[:,['id', 'comment_text', 'threat']]\n",
    "insult_df = data.loc[:,['id', 'comment_text', 'insult']]\n",
    "id_hate_df = data.loc[:,['id', 'comment_text', 'identity_hate']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'toxic': 15294,\n",
       " 'severe_toxic': 1595,\n",
       " 'obscene': 8449,\n",
       " 'threat': 478,\n",
       " 'insult': 7877,\n",
       " 'identity_hate': 1405}"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toxic_features = ['toxic', 'severe_toxic', 'obscene', 'threat','insult', 'identity_hate']\n",
    "toxic_val = [data[e].value_counts()[1] for e in toxic_features]\n",
    "dict_toxic_value = {k:v for k,v in zip(toxic_features, toxic_val)}\n",
    "dict_toxic_value"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "toxic_1 = toxic_df[toxic_df['toxic'] == 1].sample(5000)\n",
    "toxic_0 = toxic_df[toxic_df['toxic'] == 0].sample(5000)\n",
    "data_toxic_balanced = pd.concat([toxic_1,toxic_0], axis = 0)\n",
    "data_toxic_balanced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3190, 3)"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "severe_toxic_1 = severe_toxic_df[severe_toxic_df['severe_toxic'] == 1].sample(1595)\n",
    "severe_toxic_0 = severe_toxic_df[severe_toxic_df['severe_toxic'] == 0].sample(1595)\n",
    "data_severe_toxic_balanced = pd.concat([severe_toxic_1,severe_toxic_0], axis = 0)\n",
    "data_severe_toxic_balanced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "obscene_1 = obscene_df[obscene_df['obscene'] == 1].sample(5000)\n",
    "obscene_0 = obscene_df[obscene_df['obscene'] == 0].sample(5000)\n",
    "data_obscene_balanced = pd.concat([obscene_1,obscene_0], axis = 0)\n",
    "data_obscene_balanced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2390, 3)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "threat_1 = threat_df[threat_df['threat'] == 1].sample(478)\n",
    "threat_0 = threat_df[obscene_df['obscene'] == 0].sample(1912)\n",
    "data_threat_balanced = pd.concat([threat_1,threat_0], axis = 0)\n",
    "data_threat_balanced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(10000, 3)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "insult_1 = insult_df[insult_df['insult'] == 1].sample(5000)\n",
    "insult_0 = insult_df[insult_df['insult'] == 0].sample(5000)\n",
    "data_insult_balanced = pd.concat([insult_1,insult_0], axis = 0)\n",
    "data_insult_balanced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7025, 3)"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_hate_1 = id_hate_df[id_hate_df['identity_hate'] == 1].sample(1405)\n",
    "id_hate_0 = id_hate_df[id_hate_df['identity_hate'] == 0].sample(5620)\n",
    "data_id_hate_balanced = pd.concat([id_hate_1,id_hate_0], axis = 0)\n",
    "data_id_hate_balanced.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_toxic_balanced.to_csv('./data/output/pre_process_data/data_toxic_balanced.csv')\n",
    "data_severe_toxic_balanced.to_csv('./data/output/pre_process_data/data_severe_toxic_balanced.csv')\n",
    "data_obscene_balanced.to_csv('./data/output/pre_process_data/data_obscene_balanced.csv')\n",
    "data_threat_balanced.to_csv('./data/output/pre_process_data/data_threat_balanced.csv')\n",
    "data_insult_balanced.to_csv('./data/output/pre_process_data/data_insult_balanced.csv')\n",
    "data_id_hate_balanced.to_csv('./data/output/pre_process_data/data_id_hate_balanced.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "import wordcloud\n",
    "from PIL import Image\n",
    "from wordcloud import WordCloud, STOPWORDS, ImageColorGenerator\n",
    "from nltk.corpus import stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def wordcloud(df, label):\n",
    "    \n",
    "    # Print only rows where the toxic category label value is 1 (ie. the comment is toxic)\n",
    "    subset=df[df[label]==1]\n",
    "    text=subset.comment_text.values\n",
    "    wc= WordCloud(background_color=\"black\",max_words=4000)\n",
    "\n",
    "    wc.generate(\" \".join(text))\n",
    "\n",
    "    plt.figure(figsize=(20,20))\n",
    "    plt.subplot(221)\n",
    "    plt.axis(\"off\")\n",
    "    plt.title(\"Words frequented in {}\".format(label), fontsize=20)\n",
    "    plt.imshow(wc.recolor(colormap= 'gist_earth' , random_state=244), alpha=0.98)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wordcloud(data_id_hate_balanced,'identity_hate')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wordcloud(data,'severe_toxic')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}