From 9b1dd69ae6ab47cec2a7f071a3f47e99d30dc829 Mon Sep 17 00:00:00 2001
From: Brian <bdeshaz@gmail.com>
Date: Sun, 7 Jun 2015 22:35:56 -0400
Subject: [PATCH] Brian's HW

---
 program_classifier.ipynb | 2144 ++++++++++++++++++++++++++++++++++++++
 1 file changed, 2144 insertions(+)
 create mode 100644 program_classifier.ipynb

diff --git a/program_classifier.ipynb b/program_classifier.ipynb
new file mode 100644
index 0000000..01010fe
--- /dev/null
+++ b/program_classifier.ipynb
@@ -0,0 +1,2144 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Make an array where:\n",
+    "* column1 is index\n",
+    "* column2 is language\n",
+    "\n",
+    "### Do this by using glob on the pro-lan dir. \n",
+    "\n",
+    "### Maybe run an Unselective Classifier to get labels"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 30,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "import glob\n",
+    "import os\n",
+    "import csv\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "import numpy as np"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "names = [os.path.basename(x) for x in glob.glob('benchmarksgame-2014-08-31/*/*/*/*')]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "names = []\n",
+    "txtf = []\n",
+    "\n",
+    "for file in glob.glob('benchmarksgame-2014-08-31/*/*/*/*/*/*'):\n",
+    "#     names.append(os.path.basename(file))\n",
+    "    names.append(os.path.splitext(file)[-2])\n",
+    "    with open(file, encoding=\"ISO-8859-1\") as some_file:\n",
+    "        txtf.append(some_file.read())\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "8410\n",
+      "8410\n"
+     ]
+    }
+   ],
+   "source": [
+    "txt = pd.Series(txtf); print(len(txtf))\n",
+    "names = pd.Series(names); print(len(names))"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "8409    benchmarksgame-2014-08-31/benchmarksgame/websi...\n",
+       "dtype: object"
+      ]
+     },
+     "execution_count": 5,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "names[-1:]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "extensions = []\n",
+    "for file in names:\n",
+    "    extensions.append(os.path.splitext(file)[-1])\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "exten2 = []\n",
+    "for file in extensions:\n",
+    "    file = re.sub('\\.','',file)\n",
+    "    exten2.append(file)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 8,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "exten = pd.Series(exten2)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df = pd.DataFrame()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "df = df.append([exten, txt]).T"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 11,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>exten</th>\n",
+       "      <th>txt</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ats</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(*&lt;/span&gt;\\n&lt;span class=\"h...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ats</td>\n",
+       "      <td>\\nWed, 23 Jan 2013 05:50:58 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>cint</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>cint</td>\n",
+       "      <td></td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;;; The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>\\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/*&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>\\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>dart</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>dart</td>\n",
+       "      <td>\\nThu, 28 Aug 2014 23:40:40 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>fpascal</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(*&lt;/span&gt;\\n&lt;span class=\"h...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>fpascal</td>\n",
+       "      <td>\\nFri, 25 Apr 2014 02:25:51 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>fsharp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(*&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>fsharp</td>\n",
+       "      <td>\\nThu, 14 Aug 2014 10:19:29 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>gcc</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>gcc</td>\n",
+       "      <td>\\nThu, 24 Apr 2014 01:10:05 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>ghc</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;--&lt;/span&gt;\\n&lt;span class=\"h...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>ghc</td>\n",
+       "      <td>\\nThu, 17 Apr 2014 00:13:33 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>gnat</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;-- The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>gnat</td>\n",
+       "      <td>\\nSat, 26 Apr 2014 16:51:41 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>go</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>go</td>\n",
+       "      <td>\\nThu, 19 Jun 2014 02:22:05 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>hipe</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;% The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>hipe</td>\n",
+       "      <td>\\nMon, 04 Mar 2013 21:40:14 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>ifc</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;! -*- mode: f90 -*-&lt;/span...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>ifc</td>\n",
+       "      <td>\\nTue, 15 Jan 2013 06:41:34 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 23:34:26 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>jruby</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language S...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>jruby</td>\n",
+       "      <td>\\nSat, 29 Mar 2014 19:58:45 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8380</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;java version \"1.8.0\"&lt;br/&gt;\\nJava(TM) SE Runt...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8381</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Mono JIT compiler version 3.8.1 (master/db3...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8382</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Dart VM version: 1.6.0 (Tue Aug 26 14:02:07...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8383</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Erlang R16B (erts-5.10.1) [source] [64-bit]...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8384</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Free Pascal Compiler version 2.6.4 [2014/03...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8385</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;F# Compiler for F# 3.1 (Open Source Edition...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8386</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;gcc version 4.8.2 (Ubuntu 4.8.2-19ubuntu1)&lt;...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8387</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;The Glorious Glasgow Haskell Compilation Sy...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8388</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;GNAT 4.6&lt;/p&gt;\\n&lt;p&gt;gcc version 4.8.2 (Ubuntu ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8389</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;go version go1.3 linux/amd64&lt;/p&gt;\\n</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8390</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;gcc version 4.8.2 (Ubuntu 4.8.2-19ubuntu1)&lt;...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8391</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;HipHop VM 3.0.0-dev (rel)&lt;br/&gt;\\nCompiler: h...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8392</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Erlang R16B (erts-5.10.1) [source] [64-bit]...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8393</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Intel(R) Fortran Intel(R) 64 Compiler XE fo...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8394</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;java version \"1.8.0\"&lt;br/&gt;\\nJava(TM) SE Runt...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8395</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;jruby 1.7.11 (1.9.3p392) 2014-02-24 86339bb...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8396</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Lua 5.1.2  Copyright (C) 1994-2007 Lua.org,...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8397</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;The OCaml native-code compiler, version 4.0...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8398</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Mozart Compiler 1.4.0 (20080704) playing Oz...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8399</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;This is perl 5, version 18, subversion 0 (v...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8400</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;PHP 5.5.0 (cli) (built: Jun 25 2013 23:17:0...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8401</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Python 3.4.0 (default, Mar 17 2014, 08:05:2...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8402</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;Welcome to Racket v6.0.&lt;/p&gt;\\n</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8403</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;ruby 1.8.7 (2008-08-11 patchlevel 72) [x86_...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8404</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;rustc 0.11.0 (aa1163b92de7717eb7c5eba002b40...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8405</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;This is SBCL 1.2.0, an implementation of AN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8406</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;java version \"1.8.0\"&lt;br/&gt;\\nJava(TM) SE Runt...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8407</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;V8 version 1.3.10 [console: dumb]&lt;/p&gt;\\n</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8408</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;VisualWorks(R) 7.7 beta2 Nov 16 2009&lt;br/&gt;\\n...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8409</th>\n",
+       "      <td></td>\n",
+       "      <td>&lt;p&gt;ruby 2.1.0p0 (2013-12-25 revision 44422) [x...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8410 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        exten                                                txt\n",
+       "0         ats  <span class=\"hl com\">(*</span>\\n<span class=\"h...\n",
+       "1         ats  \\nWed, 23 Jan 2013 05:50:58 GMT\\n\\nMAKE:\\n/usr...\n",
+       "2        cint  <span class=\"hl com\">/* The Computer Language ...\n",
+       "3        cint                                                   \n",
+       "4     clojure  <span class=\"hl slc\">;; The Computer Language ...\n",
+       "5     clojure  \\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...\n",
+       "6      csharp  <span class=\"hl com\">/*</span>\\n<span class...\n",
+       "7      csharp  \\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...\n",
+       "8        dart  <span class=\"hl com\">/* The Computer Language ...\n",
+       "9        dart  \\nThu, 28 Aug 2014 23:40:40 GMT\\n\\nCOMMAND LIN...\n",
+       "10    fpascal  <span class=\"hl com\">(*</span>\\n<span class=\"h...\n",
+       "11    fpascal  \\nFri, 25 Apr 2014 02:25:51 GMT\\n\\nMAKE:\\nmv b...\n",
+       "12     fsharp  <span class=\"hl com\">(*</span>\\n<span class...\n",
+       "13     fsharp  \\nThu, 14 Aug 2014 10:19:29 GMT\\n\\nMAKE:\\nmv b...\n",
+       "14        gcc  <span class=\"hl com\">/* The Computer Language ...\n",
+       "15        gcc  \\nThu, 24 Apr 2014 01:10:05 GMT\\n\\nMAKE:\\n/usr...\n",
+       "16        ghc  <span class=\"hl slc\">--</span>\\n<span class=\"h...\n",
+       "17        ghc  \\nThu, 17 Apr 2014 00:13:33 GMT\\n\\nMAKE:\\nmv b...\n",
+       "18       gnat  <span class=\"hl slc\">-- The Computer Language ...\n",
+       "19       gnat  \\nSat, 26 Apr 2014 16:51:41 GMT\\n\\nMAKE:\\n/usr...\n",
+       "20         go  <span class=\"hl com\">/* The Computer Language ...\n",
+       "21         go  \\nThu, 19 Jun 2014 02:22:05 GMT\\n\\nMAKE:\\n/usr...\n",
+       "22       hipe  <span class=\"hl slc\">% The Computer Language B...\n",
+       "23       hipe  \\nMon, 04 Mar 2013 21:40:14 GMT\\n\\nMAKE:\\nmv b...\n",
+       "24        ifc  <span class=\"hl slc\">! -*- mode: f90 -*-</span...\n",
+       "25        ifc  \\nTue, 15 Jan 2013 06:41:34 GMT\\n\\nMAKE:\\n/usr...\n",
+       "26       java  <span class=\"hl com\">/* The Computer Language ...\n",
+       "27       java  \\nTue, 18 Mar 2014 23:34:26 GMT\\n\\nMAKE:\\nmv b...\n",
+       "28      jruby  <span class=\"hl slc\"># The Computer Language S...\n",
+       "29      jruby  \\nSat, 29 Mar 2014 19:58:45 GMT\\n\\nMAKE:\\nmv b...\n",
+       "...       ...                                                ...\n",
+       "8380           <p>java version \"1.8.0\"<br/>\\nJava(TM) SE Runt...\n",
+       "8381           <p>Mono JIT compiler version 3.8.1 (master/db3...\n",
+       "8382           <p>Dart VM version: 1.6.0 (Tue Aug 26 14:02:07...\n",
+       "8383           <p>Erlang R16B (erts-5.10.1) [source] [64-bit]...\n",
+       "8384           <p>Free Pascal Compiler version 2.6.4 [2014/03...\n",
+       "8385           <p>F# Compiler for F# 3.1 (Open Source Edition...\n",
+       "8386           <p>gcc version 4.8.2 (Ubuntu 4.8.2-19ubuntu1)<...\n",
+       "8387           <p>The Glorious Glasgow Haskell Compilation Sy...\n",
+       "8388           <p>GNAT 4.6</p>\\n<p>gcc version 4.8.2 (Ubuntu ...\n",
+       "8389                       <p>go version go1.3 linux/amd64</p>\\n\n",
+       "8390           <p>gcc version 4.8.2 (Ubuntu 4.8.2-19ubuntu1)<...\n",
+       "8391           <p>HipHop VM 3.0.0-dev (rel)<br/>\\nCompiler: h...\n",
+       "8392           <p>Erlang R16B (erts-5.10.1) [source] [64-bit]...\n",
+       "8393           <p>Intel(R) Fortran Intel(R) 64 Compiler XE fo...\n",
+       "8394           <p>java version \"1.8.0\"<br/>\\nJava(TM) SE Runt...\n",
+       "8395           <p>jruby 1.7.11 (1.9.3p392) 2014-02-24 86339bb...\n",
+       "8396           <p>Lua 5.1.2  Copyright (C) 1994-2007 Lua.org,...\n",
+       "8397           <p>The OCaml native-code compiler, version 4.0...\n",
+       "8398           <p>Mozart Compiler 1.4.0 (20080704) playing Oz...\n",
+       "8399           <p>This is perl 5, version 18, subversion 0 (v...\n",
+       "8400           <p>PHP 5.5.0 (cli) (built: Jun 25 2013 23:17:0...\n",
+       "8401           <p>Python 3.4.0 (default, Mar 17 2014, 08:05:2...\n",
+       "8402                            <p>Welcome to Racket v6.0.</p>\\n\n",
+       "8403           <p>ruby 1.8.7 (2008-08-11 patchlevel 72) [x86_...\n",
+       "8404           <p>rustc 0.11.0 (aa1163b92de7717eb7c5eba002b40...\n",
+       "8405           <p>This is SBCL 1.2.0, an implementation of AN...\n",
+       "8406           <p>java version \"1.8.0\"<br/>\\nJava(TM) SE Runt...\n",
+       "8407                  <p>V8 version 1.3.10 [console: dumb]</p>\\n\n",
+       "8408           <p>VisualWorks(R) 7.7 beta2 Nov 16 2009<br/>\\n...\n",
+       "8409           <p>ruby 2.1.0p0 (2013-12-25 revision 44422) [x...\n",
+       "\n",
+       "[8410 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 11,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.columns = ['exten', 'txt']; df"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 12,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "df['exten'].replace('', np.nan, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 13,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "df['txt'].replace('',np.nan, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 14,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>exten</th>\n",
+       "      <th>txt</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>ats</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(*&lt;/span&gt;\\n&lt;span class=\"h...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>ats</td>\n",
+       "      <td>\\nWed, 23 Jan 2013 05:50:58 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>cint</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;;; The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>\\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/*&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>\\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>dart</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>dart</td>\n",
+       "      <td>\\nThu, 28 Aug 2014 23:40:40 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>10</th>\n",
+       "      <td>fpascal</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(*&lt;/span&gt;\\n&lt;span class=\"h...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>11</th>\n",
+       "      <td>fpascal</td>\n",
+       "      <td>\\nFri, 25 Apr 2014 02:25:51 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>12</th>\n",
+       "      <td>fsharp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(*&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>13</th>\n",
+       "      <td>fsharp</td>\n",
+       "      <td>\\nThu, 14 Aug 2014 10:19:29 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>14</th>\n",
+       "      <td>gcc</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>15</th>\n",
+       "      <td>gcc</td>\n",
+       "      <td>\\nThu, 24 Apr 2014 01:10:05 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>16</th>\n",
+       "      <td>ghc</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;--&lt;/span&gt;\\n&lt;span class=\"h...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>17</th>\n",
+       "      <td>ghc</td>\n",
+       "      <td>\\nThu, 17 Apr 2014 00:13:33 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>18</th>\n",
+       "      <td>gnat</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;-- The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>19</th>\n",
+       "      <td>gnat</td>\n",
+       "      <td>\\nSat, 26 Apr 2014 16:51:41 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>20</th>\n",
+       "      <td>go</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>21</th>\n",
+       "      <td>go</td>\n",
+       "      <td>\\nThu, 19 Jun 2014 02:22:05 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>22</th>\n",
+       "      <td>hipe</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;% The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>23</th>\n",
+       "      <td>hipe</td>\n",
+       "      <td>\\nMon, 04 Mar 2013 21:40:14 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>24</th>\n",
+       "      <td>ifc</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;! -*- mode: f90 -*-&lt;/span...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>25</th>\n",
+       "      <td>ifc</td>\n",
+       "      <td>\\nTue, 15 Jan 2013 06:41:34 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 23:34:26 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>jruby</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language S...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>jruby</td>\n",
+       "      <td>\\nSat, 29 Mar 2014 19:58:45 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>30</th>\n",
+       "      <td>ocaml</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8346</th>\n",
+       "      <td>fsharp</td>\n",
+       "      <td>\\nWed, 13 Aug 2014 23:53:57 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8347</th>\n",
+       "      <td>gcc</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8348</th>\n",
+       "      <td>gcc</td>\n",
+       "      <td>\\nSun, 20 Apr 2014 19:45:51 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8349</th>\n",
+       "      <td>gnat</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;-- The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8350</th>\n",
+       "      <td>gnat</td>\n",
+       "      <td>\\nSun, 27 Apr 2014 03:03:03 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8351</th>\n",
+       "      <td>go</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8352</th>\n",
+       "      <td>go</td>\n",
+       "      <td>\\nThu, 14 Jun 2012 22:19:30 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8353</th>\n",
+       "      <td>gpp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/*&lt;/span&gt;\\n&lt;span class=\"h...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8354</th>\n",
+       "      <td>gpp</td>\n",
+       "      <td>\\nSun, 20 Apr 2014 19:33:58 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8355</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8356</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 23:07:24 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8357</th>\n",
+       "      <td>javaxint</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8358</th>\n",
+       "      <td>javaxint</td>\n",
+       "      <td>\\nSat, 05 Mar 2011 12:09:56 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8359</th>\n",
+       "      <td>gnat</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;-- The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8360</th>\n",
+       "      <td>gnat</td>\n",
+       "      <td>\\nSun, 27 Apr 2014 03:11:21 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8361</th>\n",
+       "      <td>go</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8362</th>\n",
+       "      <td>go</td>\n",
+       "      <td>\\nThu, 19 Jun 2014 06:44:38 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8363</th>\n",
+       "      <td>gpp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/*\\n&lt;/span&gt;\\n&lt;span class=...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8364</th>\n",
+       "      <td>gpp</td>\n",
+       "      <td>\\nSun, 20 Apr 2014 19:37:12 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8365</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8366</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 23:05:24 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8367</th>\n",
+       "      <td>javaxint</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8368</th>\n",
+       "      <td>javaxint</td>\n",
+       "      <td>\\nSat, 05 Mar 2011 11:58:19 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8369</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8370</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 22:50:47 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8371</th>\n",
+       "      <td>javaxint</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8372</th>\n",
+       "      <td>javaxint</td>\n",
+       "      <td>\\nSat, 05 Mar 2011 12:25:54 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8373</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**\\n&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8374</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 22:51:36 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8377</th>\n",
+       "      <td>csv</td>\n",
+       "      <td>BZh91AY&amp;SY‘bpy\u0002p\u000b",
+       "]€x\u0010Beà\u0010\b?ÿßpaµÞ±J¢©J)T\u0014 (\u0000\u0005...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>8275 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "         exten                                                txt\n",
+       "0          ats  <span class=\"hl com\">(*</span>\\n<span class=\"h...\n",
+       "1          ats  \\nWed, 23 Jan 2013 05:50:58 GMT\\n\\nMAKE:\\n/usr...\n",
+       "2         cint  <span class=\"hl com\">/* The Computer Language ...\n",
+       "4      clojure  <span class=\"hl slc\">;; The Computer Language ...\n",
+       "5      clojure  \\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...\n",
+       "6       csharp  <span class=\"hl com\">/*</span>\\n<span class...\n",
+       "7       csharp  \\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...\n",
+       "8         dart  <span class=\"hl com\">/* The Computer Language ...\n",
+       "9         dart  \\nThu, 28 Aug 2014 23:40:40 GMT\\n\\nCOMMAND LIN...\n",
+       "10     fpascal  <span class=\"hl com\">(*</span>\\n<span class=\"h...\n",
+       "11     fpascal  \\nFri, 25 Apr 2014 02:25:51 GMT\\n\\nMAKE:\\nmv b...\n",
+       "12      fsharp  <span class=\"hl com\">(*</span>\\n<span class...\n",
+       "13      fsharp  \\nThu, 14 Aug 2014 10:19:29 GMT\\n\\nMAKE:\\nmv b...\n",
+       "14         gcc  <span class=\"hl com\">/* The Computer Language ...\n",
+       "15         gcc  \\nThu, 24 Apr 2014 01:10:05 GMT\\n\\nMAKE:\\n/usr...\n",
+       "16         ghc  <span class=\"hl slc\">--</span>\\n<span class=\"h...\n",
+       "17         ghc  \\nThu, 17 Apr 2014 00:13:33 GMT\\n\\nMAKE:\\nmv b...\n",
+       "18        gnat  <span class=\"hl slc\">-- The Computer Language ...\n",
+       "19        gnat  \\nSat, 26 Apr 2014 16:51:41 GMT\\n\\nMAKE:\\n/usr...\n",
+       "20          go  <span class=\"hl com\">/* The Computer Language ...\n",
+       "21          go  \\nThu, 19 Jun 2014 02:22:05 GMT\\n\\nMAKE:\\n/usr...\n",
+       "22        hipe  <span class=\"hl slc\">% The Computer Language B...\n",
+       "23        hipe  \\nMon, 04 Mar 2013 21:40:14 GMT\\n\\nMAKE:\\nmv b...\n",
+       "24         ifc  <span class=\"hl slc\">! -*- mode: f90 -*-</span...\n",
+       "25         ifc  \\nTue, 15 Jan 2013 06:41:34 GMT\\n\\nMAKE:\\n/usr...\n",
+       "26        java  <span class=\"hl com\">/* The Computer Language ...\n",
+       "27        java  \\nTue, 18 Mar 2014 23:34:26 GMT\\n\\nMAKE:\\nmv b...\n",
+       "28       jruby  <span class=\"hl slc\"># The Computer Language S...\n",
+       "29       jruby  \\nSat, 29 Mar 2014 19:58:45 GMT\\n\\nMAKE:\\nmv b...\n",
+       "30       ocaml  <span class=\"hl com\">(* The Computer Language ...\n",
+       "...        ...                                                ...\n",
+       "8346    fsharp  \\nWed, 13 Aug 2014 23:53:57 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8347       gcc  <span class=\"hl com\">/* The Computer Language ...\n",
+       "8348       gcc  \\nSun, 20 Apr 2014 19:45:51 GMT\\n\\nMAKE:\\n/usr...\n",
+       "8349      gnat  <span class=\"hl slc\">-- The Computer Language ...\n",
+       "8350      gnat  \\nSun, 27 Apr 2014 03:03:03 GMT\\n\\nMAKE:\\n/usr...\n",
+       "8351        go  <span class=\"hl com\">/* The Computer Language ...\n",
+       "8352        go  \\nThu, 14 Jun 2012 22:19:30 GMT\\n\\nMAKE:\\n/usr...\n",
+       "8353       gpp  <span class=\"hl com\">/*</span>\\n<span class=\"h...\n",
+       "8354       gpp  \\nSun, 20 Apr 2014 19:33:58 GMT\\n\\nMAKE:\\n/usr...\n",
+       "8355      java  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8356      java  \\nTue, 18 Mar 2014 23:07:24 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8357  javaxint  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8358  javaxint  \\nSat, 05 Mar 2011 12:09:56 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8359      gnat  <span class=\"hl slc\">-- The Computer Language ...\n",
+       "8360      gnat  \\nSun, 27 Apr 2014 03:11:21 GMT\\n\\nMAKE:\\n/usr...\n",
+       "8361        go  <span class=\"hl com\">/* The Computer Language ...\n",
+       "8362        go  \\nThu, 19 Jun 2014 06:44:38 GMT\\n\\nMAKE:\\n/usr...\n",
+       "8363       gpp  <span class=\"hl com\">/*\\n</span>\\n<span class=...\n",
+       "8364       gpp  \\nSun, 20 Apr 2014 19:37:12 GMT\\n\\nMAKE:\\n/usr...\n",
+       "8365      java  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8366      java  \\nTue, 18 Mar 2014 23:05:24 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8367  javaxint  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8368  javaxint  \\nSat, 05 Mar 2011 11:58:19 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8369      java  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8370      java  \\nTue, 18 Mar 2014 22:50:47 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8371  javaxint  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8372  javaxint  \\nSat, 05 Mar 2011 12:25:54 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8373      java  <span class=\"hl com\">/**\\n</span>\\n<span class...\n",
+       "8374      java  \\nTue, 18 Mar 2014 22:51:36 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8377       csv  BZh91AY&SY‘bpy\u0002p\n",
+       "]€x\u0010Beà\u0010\b?ÿßpaµÞ±J¢©J)T\u0014 (\u0000\u0005...\n",
+       "\n",
+       "[8275 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 14,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.dropna()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 15,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "adict = {item for item in exten}    "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 16,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "useless = ['','clean','ats','csv','fpascal','fsharp','gcc','ghc','gnat','go','gpp','hipe','ifc','javasteady','javaxint','lua','luajit','mzscheme','oz','tracemonkey','v8','vw']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 17,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "for item in useless:\n",
+    "    df = df[df.exten != item]\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 18,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>exten</th>\n",
+       "      <th>txt</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>cint</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;;; The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>\\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/*&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>\\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8</th>\n",
+       "      <td>dart</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>9</th>\n",
+       "      <td>dart</td>\n",
+       "      <td>\\nThu, 28 Aug 2014 23:40:40 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>26</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>27</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 23:34:26 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>28</th>\n",
+       "      <td>jruby</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language S...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>29</th>\n",
+       "      <td>jruby</td>\n",
+       "      <td>\\nSat, 29 Mar 2014 19:58:45 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>30</th>\n",
+       "      <td>ocaml</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>31</th>\n",
+       "      <td>ocaml</td>\n",
+       "      <td>\\nThu, 12 Sep 2013 20:23:23 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>34</th>\n",
+       "      <td>perl</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>35</th>\n",
+       "      <td>perl</td>\n",
+       "      <td>\\nTue, 21 May 2013 17:44:06 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>36</th>\n",
+       "      <td>php</td>\n",
+       "      <td>&lt;span class=\"hl opt\"&gt;&amp;lt;&lt;/span&gt;?php \\n&lt;span c...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>37</th>\n",
+       "      <td>php</td>\n",
+       "      <td>\\nTue, 25 Jun 2013 20:43:25 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>38</th>\n",
+       "      <td>racket</td>\n",
+       "      <td>&lt;span class=\"hl opt\"&gt;#&lt;/span&gt;lang racket&lt;span ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>39</th>\n",
+       "      <td>racket</td>\n",
+       "      <td>\\nFri, 28 Feb 2014 04:34:37 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>40</th>\n",
+       "      <td>rust</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;// The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>41</th>\n",
+       "      <td>rust</td>\n",
+       "      <td>\\nTue, 08 Jul 2014 00:10:57 GMT\\n\\nMAKE:\\n/usr...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>42</th>\n",
+       "      <td>sbcl</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;;;   The Computer Languag...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>43</th>\n",
+       "      <td>sbcl</td>\n",
+       "      <td>\\nSat, 31 May 2014 04:02:44 GMT\\n\\nMAKE:\\ncp: ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>44</th>\n",
+       "      <td>scala</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>45</th>\n",
+       "      <td>scala</td>\n",
+       "      <td>\\nWed, 19 Mar 2014 08:28:18 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>50</th>\n",
+       "      <td>yarv</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>51</th>\n",
+       "      <td>yarv</td>\n",
+       "      <td>\\nTue, 07 Jan 2014 21:26:00 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>52</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;;; The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>53</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>\\nFri, 18 Apr 2014 21:23:34 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>54</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>...</th>\n",
+       "      <td>...</td>\n",
+       "      <td>...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8301</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Langua...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8302</th>\n",
+       "      <td>csharp</td>\n",
+       "      <td>\\nWed, 13 Aug 2014 21:59:29 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8311</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8312</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 22:57:55 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8315</th>\n",
+       "      <td>jruby</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8316</th>\n",
+       "      <td>jruby</td>\n",
+       "      <td>\\nSun, 30 Mar 2014 03:39:44 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8317</th>\n",
+       "      <td>ocaml</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8318</th>\n",
+       "      <td>ocaml</td>\n",
+       "      <td>\\nFri, 13 Sep 2013 03:27:18 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8319</th>\n",
+       "      <td>python3</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8320</th>\n",
+       "      <td>python3</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 06:12:43 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8321</th>\n",
+       "      <td>sbcl</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;;;; The Computer Language...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8322</th>\n",
+       "      <td>sbcl</td>\n",
+       "      <td>\\nSun, 01 Jun 2014 01:39:42 GMT\\n\\nMAKE:\\ncp: ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8323</th>\n",
+       "      <td>yarv</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8324</th>\n",
+       "      <td>yarv</td>\n",
+       "      <td>\\nThu, 09 Jan 2014 05:13:53 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8335</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8336</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 22:59:09 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8339</th>\n",
+       "      <td>ocaml</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;(* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8340</th>\n",
+       "      <td>ocaml</td>\n",
+       "      <td>\\nFri, 13 Sep 2013 03:13:05 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8341</th>\n",
+       "      <td>perl</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8342</th>\n",
+       "      <td>perl</td>\n",
+       "      <td>\\nTue, 21 May 2013 09:44:09 GMT\\n\\nCOMMAND LIN...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8343</th>\n",
+       "      <td>python3</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;# The Computer Language B...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8344</th>\n",
+       "      <td>python3</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 06:18:49 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8355</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8356</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 23:07:24 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8365</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8366</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 23:05:24 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8369</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**&lt;/span&gt;\\n&lt;span class=\"...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8370</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 22:50:47 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8373</th>\n",
+       "      <td>java</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/**\\n&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>8374</th>\n",
+       "      <td>java</td>\n",
+       "      <td>\\nTue, 18 Mar 2014 22:51:36 GMT\\n\\nMAKE:\\nmv t...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "<p>4565 rows × 2 columns</p>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "        exten                                                txt\n",
+       "2        cint  <span class=\"hl com\">/* The Computer Language ...\n",
+       "4     clojure  <span class=\"hl slc\">;; The Computer Language ...\n",
+       "5     clojure  \\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...\n",
+       "6      csharp  <span class=\"hl com\">/*</span>\\n<span class...\n",
+       "7      csharp  \\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...\n",
+       "8        dart  <span class=\"hl com\">/* The Computer Language ...\n",
+       "9        dart  \\nThu, 28 Aug 2014 23:40:40 GMT\\n\\nCOMMAND LIN...\n",
+       "26       java  <span class=\"hl com\">/* The Computer Language ...\n",
+       "27       java  \\nTue, 18 Mar 2014 23:34:26 GMT\\n\\nMAKE:\\nmv b...\n",
+       "28      jruby  <span class=\"hl slc\"># The Computer Language S...\n",
+       "29      jruby  \\nSat, 29 Mar 2014 19:58:45 GMT\\n\\nMAKE:\\nmv b...\n",
+       "30      ocaml  <span class=\"hl com\">(* The Computer Language ...\n",
+       "31      ocaml  \\nThu, 12 Sep 2013 20:23:23 GMT\\n\\nMAKE:\\nmv b...\n",
+       "34       perl  <span class=\"hl slc\"># The Computer Language B...\n",
+       "35       perl  \\nTue, 21 May 2013 17:44:06 GMT\\n\\nCOMMAND LIN...\n",
+       "36        php  <span class=\"hl opt\">&lt;</span>?php \\n<span c...\n",
+       "37        php  \\nTue, 25 Jun 2013 20:43:25 GMT\\n\\nCOMMAND LIN...\n",
+       "38     racket  <span class=\"hl opt\">#</span>lang racket<span ...\n",
+       "39     racket  \\nFri, 28 Feb 2014 04:34:37 GMT\\n\\nCOMMAND LIN...\n",
+       "40       rust  <span class=\"hl slc\">// The Computer Language ...\n",
+       "41       rust  \\nTue, 08 Jul 2014 00:10:57 GMT\\n\\nMAKE:\\n/usr...\n",
+       "42       sbcl  <span class=\"hl slc\">;;   The Computer Languag...\n",
+       "43       sbcl  \\nSat, 31 May 2014 04:02:44 GMT\\n\\nMAKE:\\ncp: ...\n",
+       "44      scala  <span class=\"hl com\">/* The Computer Language ...\n",
+       "45      scala  \\nWed, 19 Mar 2014 08:28:18 GMT\\n\\nMAKE:\\nmv b...\n",
+       "50       yarv  <span class=\"hl slc\"># The Computer Language B...\n",
+       "51       yarv  \\nTue, 07 Jan 2014 21:26:00 GMT\\n\\nCOMMAND LIN...\n",
+       "52    clojure  <span class=\"hl slc\">;; The Computer Language ...\n",
+       "53    clojure  \\nFri, 18 Apr 2014 21:23:34 GMT\\n\\nMAKE:\\nmv b...\n",
+       "54     csharp  <span class=\"hl com\">/* The Computer Language ...\n",
+       "...       ...                                                ...\n",
+       "8301   csharp  <span class=\"hl com\">/* The Computer Langua...\n",
+       "8302   csharp  \\nWed, 13 Aug 2014 21:59:29 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8311     java  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8312     java  \\nTue, 18 Mar 2014 22:57:55 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8315    jruby  <span class=\"hl slc\"># The Computer Language B...\n",
+       "8316    jruby  \\nSun, 30 Mar 2014 03:39:44 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8317    ocaml  <span class=\"hl com\">(* The Computer Language ...\n",
+       "8318    ocaml  \\nFri, 13 Sep 2013 03:27:18 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8319  python3  <span class=\"hl slc\"># The Computer Language B...\n",
+       "8320  python3  \\nTue, 18 Mar 2014 06:12:43 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8321     sbcl  <span class=\"hl slc\">;;; The Computer Language...\n",
+       "8322     sbcl  \\nSun, 01 Jun 2014 01:39:42 GMT\\n\\nMAKE:\\ncp: ...\n",
+       "8323     yarv  <span class=\"hl slc\"># The Computer Language B...\n",
+       "8324     yarv  \\nThu, 09 Jan 2014 05:13:53 GMT\\n\\nCOMMAND LIN...\n",
+       "8335     java  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8336     java  \\nTue, 18 Mar 2014 22:59:09 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8339    ocaml  <span class=\"hl com\">(* The Computer Language ...\n",
+       "8340    ocaml  \\nFri, 13 Sep 2013 03:13:05 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8341     perl  <span class=\"hl slc\"># The Computer Language B...\n",
+       "8342     perl  \\nTue, 21 May 2013 09:44:09 GMT\\n\\nCOMMAND LIN...\n",
+       "8343  python3  <span class=\"hl slc\"># The Computer Language B...\n",
+       "8344  python3  \\nTue, 18 Mar 2014 06:18:49 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8355     java  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8356     java  \\nTue, 18 Mar 2014 23:07:24 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8365     java  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8366     java  \\nTue, 18 Mar 2014 23:05:24 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8369     java  <span class=\"hl com\">/**</span>\\n<span class=\"...\n",
+       "8370     java  \\nTue, 18 Mar 2014 22:50:47 GMT\\n\\nMAKE:\\nmv t...\n",
+       "8373     java  <span class=\"hl com\">/**\\n</span>\\n<span class...\n",
+       "8374     java  \\nTue, 18 Mar 2014 22:51:36 GMT\\n\\nMAKE:\\nmv t...\n",
+       "\n",
+       "[4565 rows x 2 columns]"
+      ]
+     },
+     "execution_count": 18,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.dropna()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 19,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "{'',\n",
+       " 'ats',\n",
+       " 'cint',\n",
+       " 'clean',\n",
+       " 'clojure',\n",
+       " 'csharp',\n",
+       " 'csv',\n",
+       " 'dart',\n",
+       " 'erlang',\n",
+       " 'fpascal',\n",
+       " 'fsharp',\n",
+       " 'gcc',\n",
+       " 'ghc',\n",
+       " 'gnat',\n",
+       " 'go',\n",
+       " 'gpp',\n",
+       " 'hack',\n",
+       " 'hipe',\n",
+       " 'ifc',\n",
+       " 'java',\n",
+       " 'javasteady',\n",
+       " 'javaxint',\n",
+       " 'jruby',\n",
+       " 'lua',\n",
+       " 'luajit',\n",
+       " 'mzscheme',\n",
+       " 'nodejs',\n",
+       " 'ocaml',\n",
+       " 'oz',\n",
+       " 'perl',\n",
+       " 'php',\n",
+       " 'pypy',\n",
+       " 'python',\n",
+       " 'python3',\n",
+       " 'racket',\n",
+       " 'rust',\n",
+       " 'sbcl',\n",
+       " 'scala',\n",
+       " 'tracemonkey',\n",
+       " 'v8',\n",
+       " 'vw',\n",
+       " 'yarv'}"
+      ]
+     },
+     "execution_count": 19,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "adict"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 20,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "sorter = {'c++':['cint','c++'],\n",
+    "          'clojure':['clojure'],\n",
+    "          'c#':['csharp'],\n",
+    "          'dart':['dart'],\n",
+    "          'erlang':['erlang'],\n",
+    "          'php':['hack','php'],\n",
+    "          'java':['java'],\n",
+    "          'javascript':['nodejs'],\n",
+    "          'ocaml':['ocaml'],\n",
+    "          'perl':['perl'],\n",
+    "          'python':['pypy','python','python3'],\n",
+    "          'scheme':['racket'],\n",
+    "          'rust':['rust'],\n",
+    "          'common_lisp':['sbcl'],\n",
+    "          'scala':['scala'],\n",
+    "          'ruby':['rb','jruby','yarv']}"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 21,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4699"
+      ]
+     },
+     "execution_count": 21,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(df.exten)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 22,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "df['exten'].replace('', np.nan, inplace=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 23,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4565"
+      ]
+     },
+     "execution_count": 23,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df = df.dropna(); len(df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "for item in df.exten:\n",
+    "    for name in sorter:\n",
+    "        if item in sorter[name]:\n",
+    "            df.exten[df.exten== item] = name"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 29,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>exten</th>\n",
+       "      <th>txt</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>c++</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;;; The Computer Language ...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>5</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>\\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>6</th>\n",
+       "      <td>c#</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/*&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>7</th>\n",
+       "      <td>c#</td>\n",
+       "      <td>\\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     exten                                                txt\n",
+       "2      c++  <span class=\"hl com\">/* The Computer Language ...\n",
+       "4  clojure  <span class=\"hl slc\">;; The Computer Language ...\n",
+       "5  clojure  \\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...\n",
+       "6       c#  <span class=\"hl com\">/*</span>\\n<span class...\n",
+       "7       c#  \\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b..."
+      ]
+     },
+     "execution_count": 29,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 48,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "def periods_feature(text):\n",
+    "    periods = text.count(\".\")\n",
+    "    return [periods/ len(text)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 49,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "def curly_feature(text):\n",
+    "    curly = text.count(\"{\")\n",
+    "    return [curly/len(text)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 50,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def longest_run_of_nonword_feature(text):\n",
+    "    runs = sorted(re.findall(r\"[\\W]+\", text), key=len)\n",
+    "    longest = runs[-1]\n",
+    "    return [len(longest)]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 58,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "def feature_vector(text):\n",
+    "    return longest_run_of_nonword_feature(text) + curly_feature(text) + periods_feature(text)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 59,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "\n",
+    "class CustomFeaturizer:\n",
+    "    def __init__(self, *featurizers):\n",
+    "        self.featurizers = featurizers\n",
+    "        \n",
+    "    def fit(self, X, y=None):\n",
+    "        return self\n",
+    "        \n",
+    "    def transform(self, X):\n",
+    "        \"\"\"Given a list of original data, return a list of feature vectors.\"\"\"\n",
+    "        fvs = []\n",
+    "        for datum in X:\n",
+    "            fv = np.array([f(datum) for f in self.featurizers])\n",
+    "            fvs.append(fv.reshape(1, -1)[0])\n",
+    "        return np.array(fvs)\n",
+    "    \n",
+    "lang_featurizer = CustomFeaturizer(longest_run_of_nonword_feature, curly_feature,\n",
+    "                                  periods_feature)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 65,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "   nonword    curly%   period%\n",
+      "0       15  0.000772  0.001029\n",
+      "1       35  0.000000  0.001311\n",
+      "2        4  0.000000  0.023649\n",
+      "3       15  0.001869  0.002419\n",
+      "4        3  0.000000  0.009447\n",
+      "5       11  0.001369  0.002464\n",
+      "6        3  0.000000  0.001855\n",
+      "7       23  0.001733  0.002460\n",
+      "8        3  0.000000  0.013064\n",
+      "9        7  0.001368  0.001563\n"
+     ]
+    }
+   ],
+   "source": [
+    "la_df = pd.DataFrame(lang_featurizer.transform(df.txt), columns=['nonword','curly%','period%']); print(la_df[:10])"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 66,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "4565"
+      ]
+     },
+     "execution_count": 66,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "len(la_df)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 82,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "test_df = df.reset_index(inplace = True, drop=True)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 92,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "test_df = pd.concat([df,la_df], axis=1)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 96,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/html": [
+       "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
+       "<table border=\"1\" class=\"dataframe\">\n",
+       "  <thead>\n",
+       "    <tr style=\"text-align: right;\">\n",
+       "      <th></th>\n",
+       "      <th>exten</th>\n",
+       "      <th>txt</th>\n",
+       "      <th>nonword</th>\n",
+       "      <th>curly%</th>\n",
+       "      <th>period%</th>\n",
+       "    </tr>\n",
+       "  </thead>\n",
+       "  <tbody>\n",
+       "    <tr>\n",
+       "      <th>0</th>\n",
+       "      <td>c++</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/* The Computer Language ...</td>\n",
+       "      <td>15</td>\n",
+       "      <td>0.000772</td>\n",
+       "      <td>0.001029</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>1</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>&lt;span class=\"hl slc\"&gt;;; The Computer Language ...</td>\n",
+       "      <td>35</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.001311</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>2</th>\n",
+       "      <td>clojure</td>\n",
+       "      <td>\\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "      <td>4</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.023649</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>3</th>\n",
+       "      <td>c#</td>\n",
+       "      <td>&lt;span class=\"hl com\"&gt;/*&lt;/span&gt;\\n&lt;span class...</td>\n",
+       "      <td>15</td>\n",
+       "      <td>0.001869</td>\n",
+       "      <td>0.002419</td>\n",
+       "    </tr>\n",
+       "    <tr>\n",
+       "      <th>4</th>\n",
+       "      <td>c#</td>\n",
+       "      <td>\\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...</td>\n",
+       "      <td>3</td>\n",
+       "      <td>0.000000</td>\n",
+       "      <td>0.009447</td>\n",
+       "    </tr>\n",
+       "  </tbody>\n",
+       "</table>\n",
+       "</div>"
+      ],
+      "text/plain": [
+       "     exten                                                txt  nonword  \\\n",
+       "0      c++  <span class=\"hl com\">/* The Computer Language ...       15   \n",
+       "1  clojure  <span class=\"hl slc\">;; The Computer Language ...       35   \n",
+       "2  clojure  \\nFri, 18 Apr 2014 21:29:20 GMT\\n\\nMAKE:\\nmv b...        4   \n",
+       "3       c#  <span class=\"hl com\">/*</span>\\n<span class...       15   \n",
+       "4       c#  \\nThu, 14 Aug 2014 10:17:17 GMT\\n\\nMAKE:\\nmv b...        3   \n",
+       "\n",
+       "     curly%   period%  \n",
+       "0  0.000772  0.001029  \n",
+       "1  0.000000  0.001311  \n",
+       "2  0.000000  0.023649  \n",
+       "3  0.001869  0.002419  \n",
+       "4  0.000000  0.009447  "
+      ]
+     },
+     "execution_count": 96,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "test_df.head()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 97,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from sklearn.svm import LinearSVC"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 98,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "svc = LinearSVC()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 100,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "analysis = test_df[:]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 102,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "del analysis['txt']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 106,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "X = analysis[['nonword','curly%','period%']]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 107,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "y = analysis['exten']"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 108,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "from sklearn.cross_validation import train_test_split"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 109,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "testX, trainX, testy, trainy = train_test_split(X,y)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 110,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
+       "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
+       "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
+       "     verbose=0)"
+      ]
+     },
+     "execution_count": 110,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "svc.fit(trainX, trainy)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 114,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.17470055506865323"
+      ]
+     },
+     "execution_count": 114,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "svc.score(testX, testy)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 116,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [],
+   "source": [
+    "from sklearn.naive_bayes import MultinomialNB"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 117,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": [
+    "mnb = MultinomialNB()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 118,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "MultinomialNB(alpha=1.0, class_prior=None, fit_prior=True)"
+      ]
+     },
+     "execution_count": 118,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mnb.fit(trainX, trainy)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 119,
+   "metadata": {
+    "collapsed": false
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "0.15103710195734735"
+      ]
+     },
+     "execution_count": 119,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "mnb.score(testX, testy)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {
+    "collapsed": true
+   },
+   "outputs": [],
+   "source": []
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.4.3"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
+}