Sentiment Analysis Classifier.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Stock Sentiment Analysis (Classifier) using Top News Headlines"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Let's import relevant libraries.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.metrics import classification_report,confusion_matrix,accuracy_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('Data.csv', encoding = \"ISO-8859-1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date</th>\n",
       "      <th>Label</th>\n",
       "      <th>Top1</th>\n",
       "      <th>Top2</th>\n",
       "      <th>Top3</th>\n",
       "      <th>Top4</th>\n",
       "      <th>Top5</th>\n",
       "      <th>Top6</th>\n",
       "      <th>Top7</th>\n",
       "      <th>Top8</th>\n",
       "      <th>...</th>\n",
       "      <th>Top16</th>\n",
       "      <th>Top17</th>\n",
       "      <th>Top18</th>\n",
       "      <th>Top19</th>\n",
       "      <th>Top20</th>\n",
       "      <th>Top21</th>\n",
       "      <th>Top22</th>\n",
       "      <th>Top23</th>\n",
       "      <th>Top24</th>\n",
       "      <th>Top25</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2000-01-03</td>\n",
       "      <td>0</td>\n",
       "      <td>A 'hindrance to operations': extracts from the...</td>\n",
       "      <td>Scorecard</td>\n",
       "      <td>Hughes' instant hit buoys Blues</td>\n",
       "      <td>Jack gets his skates on at ice-cold Alex</td>\n",
       "      <td>Chaos as Maracana builds up for United</td>\n",
       "      <td>Depleted Leicester prevail as Elliott spoils E...</td>\n",
       "      <td>Hungry Spurs sense rich pickings</td>\n",
       "      <td>Gunners so wide of an easy target</td>\n",
       "      <td>...</td>\n",
       "      <td>Flintoff injury piles on woe for England</td>\n",
       "      <td>Hunters threaten Jospin with new battle of the...</td>\n",
       "      <td>Kohl's successor drawn into scandal</td>\n",
       "      <td>The difference between men and women</td>\n",
       "      <td>Sara Denver, nurse turned solicitor</td>\n",
       "      <td>Diana's landmine crusade put Tories in a panic</td>\n",
       "      <td>Yeltsin's resignation caught opposition flat-f...</td>\n",
       "      <td>Russian roulette</td>\n",
       "      <td>Sold out</td>\n",
       "      <td>Recovering a title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2000-01-04</td>\n",
       "      <td>0</td>\n",
       "      <td>Scorecard</td>\n",
       "      <td>The best lake scene</td>\n",
       "      <td>Leader: German sleaze inquiry</td>\n",
       "      <td>Cheerio, boyo</td>\n",
       "      <td>The main recommendations</td>\n",
       "      <td>Has Cubie killed fees?</td>\n",
       "      <td>Has Cubie killed fees?</td>\n",
       "      <td>Has Cubie killed fees?</td>\n",
       "      <td>...</td>\n",
       "      <td>On the critical list</td>\n",
       "      <td>The timing of their lives</td>\n",
       "      <td>Dear doctor</td>\n",
       "      <td>Irish court halts IRA man's extradition to Nor...</td>\n",
       "      <td>Burundi peace initiative fades after rebels re...</td>\n",
       "      <td>PE points the way forward to the ECB</td>\n",
       "      <td>Campaigners keep up pressure on Nazi war crime...</td>\n",
       "      <td>Jane Ratcliffe</td>\n",
       "      <td>Yet more things you wouldn't know without the ...</td>\n",
       "      <td>Millennium bug fails to bite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2000-01-05</td>\n",
       "      <td>0</td>\n",
       "      <td>Coventry caught on counter by Flo</td>\n",
       "      <td>United's rivals on the road to Rio</td>\n",
       "      <td>Thatcher issues defence before trial by video</td>\n",
       "      <td>Police help Smith lay down the law at Everton</td>\n",
       "      <td>Tale of Trautmann bears two more retellings</td>\n",
       "      <td>England on the rack</td>\n",
       "      <td>Pakistan retaliate with call for video of Walsh</td>\n",
       "      <td>Cullinan continues his Cape monopoly</td>\n",
       "      <td>...</td>\n",
       "      <td>South Melbourne (Australia)</td>\n",
       "      <td>Necaxa (Mexico)</td>\n",
       "      <td>Real Madrid (Spain)</td>\n",
       "      <td>Raja Casablanca (Morocco)</td>\n",
       "      <td>Corinthians (Brazil)</td>\n",
       "      <td>Tony's pet project</td>\n",
       "      <td>Al Nassr (Saudi Arabia)</td>\n",
       "      <td>Ideal Holmes show</td>\n",
       "      <td>Pinochet leaves hospital after tests</td>\n",
       "      <td>Useful links</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2000-01-06</td>\n",
       "      <td>1</td>\n",
       "      <td>Pilgrim knows how to progress</td>\n",
       "      <td>Thatcher facing ban</td>\n",
       "      <td>McIlroy calls for Irish fighting spirit</td>\n",
       "      <td>Leicester bin stadium blueprint</td>\n",
       "      <td>United braced for Mexican wave</td>\n",
       "      <td>Auntie back in fashion, even if the dress look...</td>\n",
       "      <td>Shoaib appeal goes to the top</td>\n",
       "      <td>Hussain hurt by 'shambles' but lays blame on e...</td>\n",
       "      <td>...</td>\n",
       "      <td>Putin admits Yeltsin quit to give him a head s...</td>\n",
       "      <td>BBC worst hit as digital TV begins to bite</td>\n",
       "      <td>How much can you pay for...</td>\n",
       "      <td>Christmas glitches</td>\n",
       "      <td>Upending a table, Chopping a line and Scoring ...</td>\n",
       "      <td>Scientific evidence 'unreliable', defence claims</td>\n",
       "      <td>Fusco wins judicial review in extradition case</td>\n",
       "      <td>Rebels thwart Russian advance</td>\n",
       "      <td>Blair orders shake-up of failing NHS</td>\n",
       "      <td>Lessons of law's hard heart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2000-01-07</td>\n",
       "      <td>1</td>\n",
       "      <td>Hitches and Horlocks</td>\n",
       "      <td>Beckham off but United survive</td>\n",
       "      <td>Breast cancer screening</td>\n",
       "      <td>Alan Parker</td>\n",
       "      <td>Guardian readers: are you all whingers?</td>\n",
       "      <td>Hollywood Beyond</td>\n",
       "      <td>Ashes and diamonds</td>\n",
       "      <td>Whingers - a formidable minority</td>\n",
       "      <td>...</td>\n",
       "      <td>Most everywhere:  UDIs</td>\n",
       "      <td>Most wanted:  Chloe lunettes</td>\n",
       "      <td>Return of the cane 'completely off the agenda'</td>\n",
       "      <td>From Sleepy Hollow to Greeneland</td>\n",
       "      <td>Blunkett outlines vision for over 11s</td>\n",
       "      <td>Embattled Dobson attacks 'play now, pay later'...</td>\n",
       "      <td>Doom and the Dome</td>\n",
       "      <td>What is the north-south divide?</td>\n",
       "      <td>Aitken released from jail</td>\n",
       "      <td>Gone aloft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 27 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         Date  Label                                               Top1  \\\n",
       "0  2000-01-03      0  A 'hindrance to operations': extracts from the...   \n",
       "1  2000-01-04      0                                          Scorecard   \n",
       "2  2000-01-05      0                  Coventry caught on counter by Flo   \n",
       "3  2000-01-06      1                      Pilgrim knows how to progress   \n",
       "4  2000-01-07      1                               Hitches and Horlocks   \n",
       "\n",
       "                                 Top2  \\\n",
       "0                           Scorecard   \n",
       "1                 The best lake scene   \n",
       "2  United's rivals on the road to Rio   \n",
       "3                 Thatcher facing ban   \n",
       "4      Beckham off but United survive   \n",
       "\n",
       "                                            Top3  \\\n",
       "0                Hughes' instant hit buoys Blues   \n",
       "1                  Leader: German sleaze inquiry   \n",
       "2  Thatcher issues defence before trial by video   \n",
       "3        McIlroy calls for Irish fighting spirit   \n",
       "4                        Breast cancer screening   \n",
       "\n",
       "                                            Top4  \\\n",
       "0       Jack gets his skates on at ice-cold Alex   \n",
       "1                                  Cheerio, boyo   \n",
       "2  Police help Smith lay down the law at Everton   \n",
       "3                Leicester bin stadium blueprint   \n",
       "4                                    Alan Parker   \n",
       "\n",
       "                                          Top5  \\\n",
       "0       Chaos as Maracana builds up for United   \n",
       "1                     The main recommendations   \n",
       "2  Tale of Trautmann bears two more retellings   \n",
       "3               United braced for Mexican wave   \n",
       "4      Guardian readers: are you all whingers?   \n",
       "\n",
       "                                                Top6  \\\n",
       "0  Depleted Leicester prevail as Elliott spoils E...   \n",
       "1                             Has Cubie killed fees?   \n",
       "2                                England on the rack   \n",
       "3  Auntie back in fashion, even if the dress look...   \n",
       "4                                   Hollywood Beyond   \n",
       "\n",
       "                                              Top7  \\\n",
       "0                 Hungry Spurs sense rich pickings   \n",
       "1                           Has Cubie killed fees?   \n",
       "2  Pakistan retaliate with call for video of Walsh   \n",
       "3                    Shoaib appeal goes to the top   \n",
       "4                               Ashes and diamonds   \n",
       "\n",
       "                                                Top8  ...  \\\n",
       "0                  Gunners so wide of an easy target  ...   \n",
       "1                             Has Cubie killed fees?  ...   \n",
       "2               Cullinan continues his Cape monopoly  ...   \n",
       "3  Hussain hurt by 'shambles' but lays blame on e...  ...   \n",
       "4                   Whingers - a formidable minority  ...   \n",
       "\n",
       "                                               Top16  \\\n",
       "0           Flintoff injury piles on woe for England   \n",
       "1                               On the critical list   \n",
       "2                        South Melbourne (Australia)   \n",
       "3  Putin admits Yeltsin quit to give him a head s...   \n",
       "4                             Most everywhere:  UDIs   \n",
       "\n",
       "                                               Top17  \\\n",
       "0  Hunters threaten Jospin with new battle of the...   \n",
       "1                          The timing of their lives   \n",
       "2                                    Necaxa (Mexico)   \n",
       "3         BBC worst hit as digital TV begins to bite   \n",
       "4                       Most wanted:  Chloe lunettes   \n",
       "\n",
       "                                            Top18  \\\n",
       "0             Kohl's successor drawn into scandal   \n",
       "1                                     Dear doctor   \n",
       "2                             Real Madrid (Spain)   \n",
       "3                     How much can you pay for...   \n",
       "4  Return of the cane 'completely off the agenda'   \n",
       "\n",
       "                                               Top19  \\\n",
       "0               The difference between men and women   \n",
       "1  Irish court halts IRA man's extradition to Nor...   \n",
       "2                          Raja Casablanca (Morocco)   \n",
       "3                                 Christmas glitches   \n",
       "4                   From Sleepy Hollow to Greeneland   \n",
       "\n",
       "                                               Top20  \\\n",
       "0                Sara Denver, nurse turned solicitor   \n",
       "1  Burundi peace initiative fades after rebels re...   \n",
       "2                               Corinthians (Brazil)   \n",
       "3  Upending a table, Chopping a line and Scoring ...   \n",
       "4              Blunkett outlines vision for over 11s   \n",
       "\n",
       "                                               Top21  \\\n",
       "0     Diana's landmine crusade put Tories in a panic   \n",
       "1               PE points the way forward to the ECB   \n",
       "2                                 Tony's pet project   \n",
       "3   Scientific evidence 'unreliable', defence claims   \n",
       "4  Embattled Dobson attacks 'play now, pay later'...   \n",
       "\n",
       "                                               Top22  \\\n",
       "0  Yeltsin's resignation caught opposition flat-f...   \n",
       "1  Campaigners keep up pressure on Nazi war crime...   \n",
       "2                            Al Nassr (Saudi Arabia)   \n",
       "3     Fusco wins judicial review in extradition case   \n",
       "4                                  Doom and the Dome   \n",
       "\n",
       "                             Top23  \\\n",
       "0                 Russian roulette   \n",
       "1                   Jane Ratcliffe   \n",
       "2                Ideal Holmes show   \n",
       "3    Rebels thwart Russian advance   \n",
       "4  What is the north-south divide?   \n",
       "\n",
       "                                               Top24  \\\n",
       "0                                           Sold out   \n",
       "1  Yet more things you wouldn't know without the ...   \n",
       "2               Pinochet leaves hospital after tests   \n",
       "3               Blair orders shake-up of failing NHS   \n",
       "4                          Aitken released from jail   \n",
       "\n",
       "                          Top25  \n",
       "0            Recovering a title  \n",
       "1  Millennium bug fails to bite  \n",
       "2                  Useful links  \n",
       "3   Lessons of law's hard heart  \n",
       "4                    Gone aloft  \n",
       "\n",
       "[5 rows x 27 columns]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(4101, 27)"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**As we can see, we have a dataset with Top 25 NEWS Headlines and a label that says either the Stock Price Increased or Decreased on a given date.**\n",
    "\n",
    "**We have 4101 records for period of year 2000 to 2016.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    2166\n",
       "0    1935\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Label.value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**As we can see that this is a pretty balanced Dataset and hence we can start next steps right away.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**As this is a timeseries data, it is recommended to consider older data in Train dataset and newer data in Test dataset.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = df[df['Date'] < '20150101']\n",
    "test = df[df['Date'] > '20141231']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Let's remove special characters as they add no values in our sentiment analysis.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A  hindrance to operations   extracts from the...</td>\n",
       "      <td>Scorecard</td>\n",
       "      <td>Hughes  instant hit buoys Blues</td>\n",
       "      <td>Jack gets his skates on at ice cold Alex</td>\n",
       "      <td>Chaos as Maracana builds up for United</td>\n",
       "      <td>Depleted Leicester prevail as Elliott spoils E...</td>\n",
       "      <td>Hungry Spurs sense rich pickings</td>\n",
       "      <td>Gunners so wide of an easy target</td>\n",
       "      <td>Derby raise a glass to Strupar s debut double</td>\n",
       "      <td>Southgate strikes  Leeds pay the penalty</td>\n",
       "      <td>...</td>\n",
       "      <td>Flintoff injury piles on woe for England</td>\n",
       "      <td>Hunters threaten Jospin with new battle of the...</td>\n",
       "      <td>Kohl s successor drawn into scandal</td>\n",
       "      <td>The difference between men and women</td>\n",
       "      <td>Sara Denver  nurse turned solicitor</td>\n",
       "      <td>Diana s landmine crusade put Tories in a panic</td>\n",
       "      <td>Yeltsin s resignation caught opposition flat f...</td>\n",
       "      <td>Russian roulette</td>\n",
       "      <td>Sold out</td>\n",
       "      <td>Recovering a title</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Scorecard</td>\n",
       "      <td>The best lake scene</td>\n",
       "      <td>Leader  German sleaze inquiry</td>\n",
       "      <td>Cheerio  boyo</td>\n",
       "      <td>The main recommendations</td>\n",
       "      <td>Has Cubie killed fees</td>\n",
       "      <td>Has Cubie killed fees</td>\n",
       "      <td>Has Cubie killed fees</td>\n",
       "      <td>Hopkins  furious  at Foster s lack of Hannibal...</td>\n",
       "      <td>Has Cubie killed fees</td>\n",
       "      <td>...</td>\n",
       "      <td>On the critical list</td>\n",
       "      <td>The timing of their lives</td>\n",
       "      <td>Dear doctor</td>\n",
       "      <td>Irish court halts IRA man s extradition to Nor...</td>\n",
       "      <td>Burundi peace initiative fades after rebels re...</td>\n",
       "      <td>PE points the way forward to the ECB</td>\n",
       "      <td>Campaigners keep up pressure on Nazi war crime...</td>\n",
       "      <td>Jane Ratcliffe</td>\n",
       "      <td>Yet more things you wouldn t know without the ...</td>\n",
       "      <td>Millennium bug fails to bite</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Coventry caught on counter by Flo</td>\n",
       "      <td>United s rivals on the road to Rio</td>\n",
       "      <td>Thatcher issues defence before trial by video</td>\n",
       "      <td>Police help Smith lay down the law at Everton</td>\n",
       "      <td>Tale of Trautmann bears two more retellings</td>\n",
       "      <td>England on the rack</td>\n",
       "      <td>Pakistan retaliate with call for video of Walsh</td>\n",
       "      <td>Cullinan continues his Cape monopoly</td>\n",
       "      <td>McGrath puts India out of their misery</td>\n",
       "      <td>Blair Witch bandwagon rolls on</td>\n",
       "      <td>...</td>\n",
       "      <td>South Melbourne  Australia</td>\n",
       "      <td>Necaxa  Mexico</td>\n",
       "      <td>Real Madrid  Spain</td>\n",
       "      <td>Raja Casablanca  Morocco</td>\n",
       "      <td>Corinthians  Brazil</td>\n",
       "      <td>Tony s pet project</td>\n",
       "      <td>Al Nassr  Saudi Arabia</td>\n",
       "      <td>Ideal Holmes show</td>\n",
       "      <td>Pinochet leaves hospital after tests</td>\n",
       "      <td>Useful links</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Pilgrim knows how to progress</td>\n",
       "      <td>Thatcher facing ban</td>\n",
       "      <td>McIlroy calls for Irish fighting spirit</td>\n",
       "      <td>Leicester bin stadium blueprint</td>\n",
       "      <td>United braced for Mexican wave</td>\n",
       "      <td>Auntie back in fashion  even if the dress look...</td>\n",
       "      <td>Shoaib appeal goes to the top</td>\n",
       "      <td>Hussain hurt by  shambles  but lays blame on e...</td>\n",
       "      <td>England s decade of disasters</td>\n",
       "      <td>Revenge is sweet for jubilant Cronje</td>\n",
       "      <td>...</td>\n",
       "      <td>Putin admits Yeltsin quit to give him a head s...</td>\n",
       "      <td>BBC worst hit as digital TV begins to bite</td>\n",
       "      <td>How much can you pay for</td>\n",
       "      <td>Christmas glitches</td>\n",
       "      <td>Upending a table  Chopping a line and Scoring ...</td>\n",
       "      <td>Scientific evidence  unreliable   defence claims</td>\n",
       "      <td>Fusco wins judicial review in extradition case</td>\n",
       "      <td>Rebels thwart Russian advance</td>\n",
       "      <td>Blair orders shake up of failing NHS</td>\n",
       "      <td>Lessons of law s hard heart</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Hitches and Horlocks</td>\n",
       "      <td>Beckham off but United survive</td>\n",
       "      <td>Breast cancer screening</td>\n",
       "      <td>Alan Parker</td>\n",
       "      <td>Guardian readers  are you all whingers</td>\n",
       "      <td>Hollywood Beyond</td>\n",
       "      <td>Ashes and diamonds</td>\n",
       "      <td>Whingers   a formidable minority</td>\n",
       "      <td>Alan Parker   part two</td>\n",
       "      <td>Thuggery  Toxins and Ties</td>\n",
       "      <td>...</td>\n",
       "      <td>Most everywhere   UDIs</td>\n",
       "      <td>Most wanted   Chloe lunettes</td>\n",
       "      <td>Return of the cane  completely off the agenda</td>\n",
       "      <td>From Sleepy Hollow to Greeneland</td>\n",
       "      <td>Blunkett outlines vision for over   s</td>\n",
       "      <td>Embattled Dobson attacks  play now  pay later ...</td>\n",
       "      <td>Doom and the Dome</td>\n",
       "      <td>What is the north south divide</td>\n",
       "      <td>Aitken released from jail</td>\n",
       "      <td>Gone aloft</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0  \\\n",
       "0  A  hindrance to operations   extracts from the...   \n",
       "1                                          Scorecard   \n",
       "2                  Coventry caught on counter by Flo   \n",
       "3                      Pilgrim knows how to progress   \n",
       "4                               Hitches and Horlocks   \n",
       "\n",
       "                                    1  \\\n",
       "0                           Scorecard   \n",
       "1                 The best lake scene   \n",
       "2  United s rivals on the road to Rio   \n",
       "3                 Thatcher facing ban   \n",
       "4      Beckham off but United survive   \n",
       "\n",
       "                                               2  \\\n",
       "0                Hughes  instant hit buoys Blues   \n",
       "1                  Leader  German sleaze inquiry   \n",
       "2  Thatcher issues defence before trial by video   \n",
       "3        McIlroy calls for Irish fighting spirit   \n",
       "4                        Breast cancer screening   \n",
       "\n",
       "                                               3  \\\n",
       "0       Jack gets his skates on at ice cold Alex   \n",
       "1                                  Cheerio  boyo   \n",
       "2  Police help Smith lay down the law at Everton   \n",
       "3                Leicester bin stadium blueprint   \n",
       "4                                    Alan Parker   \n",
       "\n",
       "                                             4  \\\n",
       "0       Chaos as Maracana builds up for United   \n",
       "1                     The main recommendations   \n",
       "2  Tale of Trautmann bears two more retellings   \n",
       "3               United braced for Mexican wave   \n",
       "4      Guardian readers  are you all whingers    \n",
       "\n",
       "                                                   5  \\\n",
       "0  Depleted Leicester prevail as Elliott spoils E...   \n",
       "1                             Has Cubie killed fees    \n",
       "2                                England on the rack   \n",
       "3  Auntie back in fashion  even if the dress look...   \n",
       "4                                   Hollywood Beyond   \n",
       "\n",
       "                                                 6  \\\n",
       "0                 Hungry Spurs sense rich pickings   \n",
       "1                           Has Cubie killed fees    \n",
       "2  Pakistan retaliate with call for video of Walsh   \n",
       "3                    Shoaib appeal goes to the top   \n",
       "4                               Ashes and diamonds   \n",
       "\n",
       "                                                   7  \\\n",
       "0                  Gunners so wide of an easy target   \n",
       "1                             Has Cubie killed fees    \n",
       "2               Cullinan continues his Cape monopoly   \n",
       "3  Hussain hurt by  shambles  but lays blame on e...   \n",
       "4                   Whingers   a formidable minority   \n",
       "\n",
       "                                                   8  \\\n",
       "0      Derby raise a glass to Strupar s debut double   \n",
       "1  Hopkins  furious  at Foster s lack of Hannibal...   \n",
       "2             McGrath puts India out of their misery   \n",
       "3                      England s decade of disasters   \n",
       "4                             Alan Parker   part two   \n",
       "\n",
       "                                          9  ...  \\\n",
       "0  Southgate strikes  Leeds pay the penalty  ...   \n",
       "1                    Has Cubie killed fees   ...   \n",
       "2            Blair Witch bandwagon rolls on  ...   \n",
       "3      Revenge is sweet for jubilant Cronje  ...   \n",
       "4                 Thuggery  Toxins and Ties  ...   \n",
       "\n",
       "                                                  15  \\\n",
       "0           Flintoff injury piles on woe for England   \n",
       "1                               On the critical list   \n",
       "2                        South Melbourne  Australia    \n",
       "3  Putin admits Yeltsin quit to give him a head s...   \n",
       "4                             Most everywhere   UDIs   \n",
       "\n",
       "                                                  16  \\\n",
       "0  Hunters threaten Jospin with new battle of the...   \n",
       "1                          The timing of their lives   \n",
       "2                                    Necaxa  Mexico    \n",
       "3         BBC worst hit as digital TV begins to bite   \n",
       "4                       Most wanted   Chloe lunettes   \n",
       "\n",
       "                                               17  \\\n",
       "0             Kohl s successor drawn into scandal   \n",
       "1                                     Dear doctor   \n",
       "2                             Real Madrid  Spain    \n",
       "3                     How much can you pay for      \n",
       "4  Return of the cane  completely off the agenda    \n",
       "\n",
       "                                                  18  \\\n",
       "0               The difference between men and women   \n",
       "1  Irish court halts IRA man s extradition to Nor...   \n",
       "2                          Raja Casablanca  Morocco    \n",
       "3                                 Christmas glitches   \n",
       "4                   From Sleepy Hollow to Greeneland   \n",
       "\n",
       "                                                  19  \\\n",
       "0                Sara Denver  nurse turned solicitor   \n",
       "1  Burundi peace initiative fades after rebels re...   \n",
       "2                               Corinthians  Brazil    \n",
       "3  Upending a table  Chopping a line and Scoring ...   \n",
       "4              Blunkett outlines vision for over   s   \n",
       "\n",
       "                                                  20  \\\n",
       "0     Diana s landmine crusade put Tories in a panic   \n",
       "1               PE points the way forward to the ECB   \n",
       "2                                 Tony s pet project   \n",
       "3   Scientific evidence  unreliable   defence claims   \n",
       "4  Embattled Dobson attacks  play now  pay later ...   \n",
       "\n",
       "                                                  21  \\\n",
       "0  Yeltsin s resignation caught opposition flat f...   \n",
       "1  Campaigners keep up pressure on Nazi war crime...   \n",
       "2                            Al Nassr  Saudi Arabia    \n",
       "3     Fusco wins judicial review in extradition case   \n",
       "4                                  Doom and the Dome   \n",
       "\n",
       "                                22  \\\n",
       "0                 Russian roulette   \n",
       "1                   Jane Ratcliffe   \n",
       "2                Ideal Holmes show   \n",
       "3    Rebels thwart Russian advance   \n",
       "4  What is the north south divide    \n",
       "\n",
       "                                                  23  \\\n",
       "0                                           Sold out   \n",
       "1  Yet more things you wouldn t know without the ...   \n",
       "2               Pinochet leaves hospital after tests   \n",
       "3               Blair orders shake up of failing NHS   \n",
       "4                          Aitken released from jail   \n",
       "\n",
       "                             24  \n",
       "0            Recovering a title  \n",
       "1  Millennium bug fails to bite  \n",
       "2                  Useful links  \n",
       "3   Lessons of law s hard heart  \n",
       "4                    Gone aloft  \n",
       "\n",
       "[5 rows x 25 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train=train.iloc[:,2:27]\n",
    "x_train.replace(\"[^a-zA-Z]\",\" \",regex=True, inplace=True)\n",
    "\n",
    "\n",
    "list1= [i for i in range(25)]\n",
    "new_Index=[str(i) for i in list1]\n",
    "x_train.columns= new_Index\n",
    "x_train.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Let's bring everything to lowecase in order to treat a letter same for both cases.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>...</th>\n",
       "      <th>15</th>\n",
       "      <th>16</th>\n",
       "      <th>17</th>\n",
       "      <th>18</th>\n",
       "      <th>19</th>\n",
       "      <th>20</th>\n",
       "      <th>21</th>\n",
       "      <th>22</th>\n",
       "      <th>23</th>\n",
       "      <th>24</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>a  hindrance to operations   extracts from the...</td>\n",
       "      <td>scorecard</td>\n",
       "      <td>hughes  instant hit buoys blues</td>\n",
       "      <td>jack gets his skates on at ice cold alex</td>\n",
       "      <td>chaos as maracana builds up for united</td>\n",
       "      <td>depleted leicester prevail as elliott spoils e...</td>\n",
       "      <td>hungry spurs sense rich pickings</td>\n",
       "      <td>gunners so wide of an easy target</td>\n",
       "      <td>derby raise a glass to strupar s debut double</td>\n",
       "      <td>southgate strikes  leeds pay the penalty</td>\n",
       "      <td>...</td>\n",
       "      <td>flintoff injury piles on woe for england</td>\n",
       "      <td>hunters threaten jospin with new battle of the...</td>\n",
       "      <td>kohl s successor drawn into scandal</td>\n",
       "      <td>the difference between men and women</td>\n",
       "      <td>sara denver  nurse turned solicitor</td>\n",
       "      <td>diana s landmine crusade put tories in a panic</td>\n",
       "      <td>yeltsin s resignation caught opposition flat f...</td>\n",
       "      <td>russian roulette</td>\n",
       "      <td>sold out</td>\n",
       "      <td>recovering a title</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1 rows × 25 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   0          1  \\\n",
       "0  a  hindrance to operations   extracts from the...  scorecard   \n",
       "\n",
       "                                 2                                         3  \\\n",
       "0  hughes  instant hit buoys blues  jack gets his skates on at ice cold alex   \n",
       "\n",
       "                                        4  \\\n",
       "0  chaos as maracana builds up for united   \n",
       "\n",
       "                                                   5  \\\n",
       "0  depleted leicester prevail as elliott spoils e...   \n",
       "\n",
       "                                  6                                  7  \\\n",
       "0  hungry spurs sense rich pickings  gunners so wide of an easy target   \n",
       "\n",
       "                                               8  \\\n",
       "0  derby raise a glass to strupar s debut double   \n",
       "\n",
       "                                          9  ...  \\\n",
       "0  southgate strikes  leeds pay the penalty  ...   \n",
       "\n",
       "                                         15  \\\n",
       "0  flintoff injury piles on woe for england   \n",
       "\n",
       "                                                  16  \\\n",
       "0  hunters threaten jospin with new battle of the...   \n",
       "\n",
       "                                    17                                    18  \\\n",
       "0  kohl s successor drawn into scandal  the difference between men and women   \n",
       "\n",
       "                                    19  \\\n",
       "0  sara denver  nurse turned solicitor   \n",
       "\n",
       "                                               20  \\\n",
       "0  diana s landmine crusade put tories in a panic   \n",
       "\n",
       "                                                  21                22  \\\n",
       "0  yeltsin s resignation caught opposition flat f...  russian roulette   \n",
       "\n",
       "         23                  24  \n",
       "0  sold out  recovering a title  \n",
       "\n",
       "[1 rows x 25 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for index in new_Index:\n",
    "    x_train[index]=x_train[index].str.lower()\n",
    "x_train.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'scorecard the best lake scene leader  german sleaze inquiry cheerio  boyo the main recommendations has cubie killed fees  has cubie killed fees  has cubie killed fees  hopkins  furious  at foster s lack of hannibal appetite has cubie killed fees  a tale of two tails i say what i like and i like what i say elbows  eyes and nipples task force to assess risk of asteroid collision how i found myself at last on the critical list the timing of their lives dear doctor irish court halts ira man s extradition to northern ireland burundi peace initiative fades after rebels reject mandela as mediator pe points the way forward to the ecb campaigners keep up pressure on nazi war crimes suspect jane ratcliffe yet more things you wouldn t know without the movies millennium bug fails to bite'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(str(x) for x in x_train.iloc[1,0:25])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "headlines_merged = []\n",
    "for row in range(0,len(x_train.index)):\n",
    "    headlines_merged.append(' '.join(str(x) for x in x_train.iloc[row,0:25]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'a  hindrance to operations   extracts from the leaked reports scorecard hughes  instant hit buoys blues jack gets his skates on at ice cold alex chaos as maracana builds up for united depleted leicester prevail as elliott spoils everton s party hungry spurs sense rich pickings gunners so wide of an easy target derby raise a glass to strupar s debut double southgate strikes  leeds pay the penalty hammers hand robson a youthful lesson saints party like it s      wear wolves have turned into lambs stump mike catches testy gough s taunt langer escapes to hit     flintoff injury piles on woe for england hunters threaten jospin with new battle of the somme kohl s successor drawn into scandal the difference between men and women sara denver  nurse turned solicitor diana s landmine crusade put tories in a panic yeltsin s resignation caught opposition flat footed russian roulette sold out recovering a title'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "headlines_merged[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Here we have joined all the 25 columns for given record and made it one for ease of process.**"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Let's now create 'Bag of Words' using CountVectorizer.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "countvector=CountVectorizer(ngram_range=(2,2))\n",
    "train_dataset=countvector.fit_transform(headlines_merged)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestClassifier(criterion='entropy', n_estimators=150)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "randomclassifier=RandomForestClassifier(criterion='entropy', n_estimators=150)\n",
    "randomclassifier.fit(train_dataset,train['Label'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Let's now repeat the same steps for Test Data.**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_transform= []\n",
    "for row in range(0,len(test.index)):\n",
    "    test_transform.append(' '.join(str(x) for x in test.iloc[row,2:27]))\n",
    "test_dataset = countvector.transform(test_transform)\n",
    "predictions = randomclassifier.predict(test_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[142  44]\n",
      " [  7 185]]\n",
      "0.8650793650793651\n",
      "              precision    recall  f1-score   support\n",
      "\n",
      "           0       0.95      0.76      0.85       186\n",
      "           1       0.81      0.96      0.88       192\n",
      "\n",
      "    accuracy                           0.87       378\n",
      "   macro avg       0.88      0.86      0.86       378\n",
      "weighted avg       0.88      0.87      0.86       378\n",
      "\n"
     ]
    }
   ],
   "source": [
    "matrix=confusion_matrix(test['Label'],predictions)\n",
    "print(matrix)\n",
    "score=accuracy_score(test['Label'],predictions)\n",
    "print(score)\n",
    "report=classification_report(test['Label'],predictions)\n",
    "print(report)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda env:py36-TF2.0]",
   "language": "python",
   "name": "conda-env-py36-TF2.0-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}