20170810Speech/text_mining.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "res = requests.get('http://www.appledaily.com.tw/realtimenews/section/new/')\n",
    "soup = BeautifulSoup(res.text, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "news = []\n",
    "for a in soup.select('.rtddt a'):\n",
    "    news.append([a.select('time')[0].text, a.select('h2')[0].text, a.select('h1')[0].text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>20:41</td>\n",
       "      <td>社會</td>\n",
       "      <td>苗栗女保險員家門遭狙殺　身中2槍6彈孔(60445)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>20:37</td>\n",
       "      <td>體育</td>\n",
       "      <td>【蘋果LIVE】林政賢6局1安7K　中信...(13478)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>20:35</td>\n",
       "      <td>國際</td>\n",
       "      <td>【更新】倫敦雙層巴士失控　撞塌商店多人受...(18273)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>20:30</td>\n",
       "      <td>生活</td>\n",
       "      <td>快樂嬤舞刀又中彈　大動干戈只為它(37)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>20:30</td>\n",
       "      <td>生活</td>\n",
       "      <td>老樹終結鎮長要移70棵樹　不斷根惹爭議(15)</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       0   1                               2\n",
       "0  20:41  社會      苗栗女保險員家門遭狙殺　身中2槍6彈孔(60445)\n",
       "1  20:37  體育  【蘋果LIVE】林政賢6局1安7K　中信...(13478)\n",
       "2  20:35  國際  【更新】倫敦雙層巴士失控　撞塌商店多人受...(18273)\n",
       "3  20:30  生活            快樂嬤舞刀又中彈　大動干戈只為它(37)\n",
       "4  20:30  生活         老樹終結鎮長要移70棵樹　不斷根惹爭議(15)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas\n",
    "df = pandas.DataFrame(news)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "df.columns = ['time', 'category', 'title']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "df.to_excel('news.xlsx')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 英文斷詞"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['this', 'is', 'a', 'book']\n",
      "['I', 'like', 'this', 'book']\n"
     ]
    }
   ],
   "source": [
    "a = 'this is a book'\n",
    "b = 'I like this book'\n",
    "print(a.split())\n",
    "print(b.split())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 安裝Jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: jieba in c:\\programdata\\anaconda3\\lib\\site-packages\n"
     ]
    }
   ],
   "source": [
    "! pip install jieba"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "那\n",
      "酸民\n",
      "婉君\n",
      "也\n",
      "可以\n",
      "報名\n",
      "嗎\n",
      "?\n"
     ]
    }
   ],
   "source": [
    "import jieba\n",
    "a = '那酸民婉君也可以報名嗎?'\n",
    "jieba.add_word('酸民')\n",
    "jieba.add_word('婉君')\n",
    "\n",
    "for ele in jieba.cut(a):\n",
    "    print(ele)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "article = '''\n",
    "同運團體伴侶盟昨提出法院應即刻准許同性伴侶結婚登記的訴求後，下一代幸福聯盟今天也發表聲明，強調同性婚姻登記於法無據，政府及各黨派應補正公投法，將婚姻定義的詮釋權還給全體國人同胞。\n",
    "以下是聲明全文：\n",
    "一、同性婚姻登記於法無據\n",
    "依司法院釋字第748號解釋，大法官並未宣告特定民法條文立即失效，僅揭示有關機關2年後若「未使相同性別二人，得為經營共同生活之目的，成立具有親密性及排他性之永久結合關係」，同性二人得為登記婚。至於當以何種形式，保障同性二人間之永久結合法律關係，大法官明示屬立法院權限。\n",
    "其次，大法官考量同性結合關係的複雜性及爭議性需較長的審議期間，特別給予立法機關二年期限。按照司法院釋字185號解釋，司法院所為之解釋有拘束全國各機關和人民的效力。因此，行政法院和戶政機關亦應受釋字748號二年立法期限的拘束，在立法配套措施完成前，全國各機關不得逕自準許同性婚姻登記，以免破壞社會秩序，產生身分關係之爭議，造成子女身分關係的不確定1。\n",
    "二、支持法官獨立審判 不受同運干擾\n",
    "按憲法第80條「法官須超出黨派以外，依據法律獨立審判，不受任何干涉。」（另法官法第13條參照）\n",
    "同運團體於法院審理程序中，大張旗鼓開記者會，恐藉媒體引導輿論使法官蒙受外界壓力，干擾法官依法審判的獨立性。幸福盟呼籲法官行使審判權時應超然獨立，勿因政治正確、輿論壓迫或不利益之對待，而踰越法官職權。\n",
    "三、呼籲政府及各黨派補正公投法  婚姻定義應還權於民\n",
    "幸福盟再次呼籲，政府及各黨派應補正公投法，將婚姻定義的詮釋權還給全體國人同胞。婚姻定義是婚姻的核心概念，婚姻定義本不宜由少數政治菁英壟斷。自從大法官於5月24日公布同婚釋憲結果，總統民調迎來歷史最低點2，已有不少政治工作者直接點名「同性婚姻」就是使政府聲望變差的部分因素。\n",
    "幸福盟勸勉執政者，務要謙卑傾聽人民捍衛一夫一妻自然婚姻的心聲。家是國之根本，政府不應輕易撼動婚姻家庭的根基。我們盼望政府能在不改變婚姻定義和不犧牲兒童利益的前提下，用有智慧的方式保障同性戀者的法律權益。\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "words = []\n",
    "for word in jieba.cut(article):\n",
    "    words.append(word)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "婚姻 12\n",
      "同性 9\n",
      "政府 6\n",
      "法官 6\n",
      "登記 4\n",
      "幸福 4\n",
      "大法官 4\n",
      "各黨 3\n",
      "補正 3\n",
      "公投法 3\n",
      "司法院 3\n",
      "二人 3\n",
      "法律 3\n",
      "立法 3\n",
      "獨立 3\n",
      "審判 3\n",
      "同運團體 2\n",
      "伴侶 2\n",
      "法院 2\n",
      "法無據 2\n",
      "定義的 2\n",
      "詮釋權 2\n",
      "還給 2\n",
      "全體 2\n",
      "國人 2\n",
      "同胞 2\n",
      "釋字 2\n",
      "748 2\n",
      "號解釋 2\n",
      "機關 2\n",
      "永久 2\n",
      "結合關 2\n",
      "保障 2\n",
      "二年 2\n",
      "期限 2\n",
      "拘束 2\n",
      "全國 2\n",
      "各機關 2\n",
      "人民 2\n",
      "身分 2\n",
      "干擾 2\n",
      "輿論 2\n",
      "政治 2\n",
      "利益 2\n",
      "呼籲 2\n",
      "提出 1\n",
      "即刻 1\n",
      "准許 1\n",
      "結婚 1\n",
      "訴求 1\n",
      "下一代 1\n",
      "聯盟 1\n",
      "今天 1\n",
      "發表聲明 1\n",
      "強調 1\n",
      "以下 1\n",
      "聲明 1\n",
      "全文 1\n",
      "並未 1\n",
      "宣告 1\n",
      "特定 1\n",
      "民法 1\n",
      "條文 1\n",
      "立即 1\n",
      "失效 1\n",
      "揭示 1\n"
     ]
    }
   ],
   "source": [
    "from collections import Counter\n",
    "c = Counter(words)\n",
    "for k, v in c.most_common(100):\n",
    "    if len(k) >= 2:\n",
    "        print(k,v)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 詞頻矩陣"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import jieba\n",
    "jieba.add_word('柯P')\n",
    "jieba.add_word('洪智坤')\n",
    "ary = ['【更新】柯P：洪智坤洩漏公文案還沒看到公文　今處理',\n",
    "       '留洪智坤 柯：殘障求職不易',\n",
    "       '人事處議處洪智坤　柯P：不清楚議處結果']\n",
    "\n",
    "corpus = []\n",
    "for title in ary:\n",
    "    corpus.append(' '.join(jieba.cut(title)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['【 更新 】 柯P ： 洪智坤 洩漏 公文 案還 沒 看到 公文 \\u3000 今處理',\n",
       " '留 洪智坤   柯 ： 殘障 求職 不易',\n",
       " '人事 處議 處 洪智坤 \\u3000 柯P ： 不 清楚 議處 結果']"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "corpus"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "不易 人事 今處理 公文 更新 柯p 案還 殘障 求職 洩漏 洪智坤 清楚 看到 結果 處議 議處\n",
      "[[0 0 1 2 1 1 1 0 0 1 1 0 1 0 0 0]\n",
      " [1 0 0 0 0 0 0 1 1 0 1 0 0 0 0 0]\n",
      " [0 1 0 0 0 1 0 0 0 0 1 1 0 1 1 1]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer() \n",
    "X = vectorizer.fit_transform(corpus)\n",
    "\n",
    "word = vectorizer.get_feature_names() \n",
    "print(' '.join(word))\n",
    "print(X.toarray())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 計算cosine distance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.          0.15075567  0.22792115]\n",
      " [ 0.15075567  1.          0.18898224]\n",
      " [ 0.22792115  0.18898224  1.        ]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "cosine_similarities = cosine_similarity(X, X)\n",
    "print(cosine_similarities)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 文章分群"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from xml.dom   import minidom\n",
    "from xml.etree import ElementTree\n",
    "import jieba.analyse, re\n",
    "\n",
    "with open('1435449602.xml', 'r', encoding='utf-8') as f:\n",
    "    events = ElementTree.fromstring(f.read())\n",
    "\n",
    "corpus = []\n",
    "ary    = []\n",
    "for elem in events.findall('./channel/item'):\n",
    "    title       = elem.find('title').text\n",
    "    description = elem.find('description').text\n",
    "    word_list = []\n",
    "    \n",
    "    ary.append(title)\n",
    "    for word in jieba.cut(description):\n",
    "        if re.match('[\\u4e00-\\u9fa5]+', word):\n",
    "            word_list.append(word)\n",
    "    corpus.append(' '.join(word_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "vectorizer = CountVectorizer() \n",
    "X = vectorizer.fit_transform(corpus)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "transformer = TfidfTransformer()\n",
    "tfidf  = transformer.fit_transform(X)\n",
    "weight = tfidf.toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(147, 12248)"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "n_cosine_similarities = cosine_similarity(tfidf, tfidf)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 1.        ,  0.10502505,  0.00455773, ...,  0.11666613,\n",
       "         0.01667092,  0.01242793],\n",
       "       [ 0.10502505,  1.        ,  0.0153138 , ...,  0.10593473,\n",
       "         0.01586742,  0.00795129],\n",
       "       [ 0.00455773,  0.0153138 ,  1.        , ...,  0.00857993,\n",
       "         0.        ,  0.        ],\n",
       "       ..., \n",
       "       [ 0.11666613,  0.10593473,  0.00857993, ...,  1.        ,\n",
       "         0.01689612,  0.01757607],\n",
       "       [ 0.01667092,  0.01586742,  0.        , ...,  0.01689612,\n",
       "         1.        ,  0.21282872],\n",
       "       [ 0.01242793,  0.00795129,  0.        , ...,  0.01757607,\n",
       "         0.21282872,  1.        ]])"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "n_cosine_similarities"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from sklearn import cluster\n",
    "c = cluster.KMeans(n_clusters=4)\n",
    "k_data = c.fit_predict(weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "title_ary = np.array(ary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "八仙塵爆  五相關人依公共危險重傷害法辦\n",
      "八仙樂園意外 病患持續增加中\n",
      "6月28日各報頭版要聞\n",
      "八仙樂園舞台大火 逾400人輕重傷\n",
      "八仙樂園意外 毛揆取消視察臺東行程\n",
      "八仙樂園貼出暫停營業海報\n",
      "八仙樂園爆炸案 專家認玉米粉危險性高應管制使用\n",
      "八仙樂園粉塵燃爆 400人傷 負責人被移送\n",
      "八仙案  新店慈濟4傷者危急\n",
      "八仙樂園爆炸案 負責人等5人移送\n",
      "八仙樂園爆炸 劣賊趁亂打劫\n",
      "侯友宜探視傷者：追究責任\n",
      "八仙樂園火警  國軍加入救援\n",
      "八仙樂園爆炸案災害應變中心　1時30分一級開設\n",
      "八仙派對彩粉釀大禍  衛福部：研議加強管理\n",
      "彩色派對主辦人：風勢太大，引燃粉塵\n",
      "八仙樂園大火  傷者查詢電話\n",
      "派對彩粉首次釀禍  蔣丙煌：研議管理\n",
      "八仙樂園火警受傷名單一覽表\n",
      "00：35統計　八仙大火重傷97人、輕傷132人\n",
      "衛福部長：北北基醫護人員全力動員\n",
      "八仙樂園粉塵瞬燃　還原失事現場影片曝光\n",
      "八仙意外毛揆取消台東熱氣球活動\n",
      "八仙樂園爆炸案 士檢派3檢察官現場指揮\n",
      "八仙樂園爆炸案 3檢察官展開調查\n",
      "八仙爆炸案   陳國恩：全力支援\n",
      "八仙大火傷患爆量  衛福調度醫院\n",
      "八仙樂園意外 洪秀柱：為傷者祈福\n",
      "陳國恩指示警方全力動員 協助救護交通順暢\n",
      "八仙樂園爆炸228人輕重傷 朱立倫：無限期停業\n",
      "八仙樂園爆炸意外 朱立倫：立即停園接受調查\n",
      "北榮33名八里傷患  2/3傷重\n",
      "八仙樂園塵爆215傷　朱立倫下令即刻停業\n",
      "八仙樂園爆炸 朱立倫4點指示\n",
      "【八仙意外】馬英九、毛治國第一時間得知　指示全\n",
      "【八仙意外】讓專業的來！柯文哲指示啟動EOC\n",
      "八仙水上樂園爆炸逾百人傷　三軍總醫院收治傷患\n"
     ]
    }
   ],
   "source": [
    "for news in title_ary[k_data == 3]:\n",
    "    print(news)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}