小说分析 - 副本.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 爬取网络小说"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 爬取多本的小说\n",
    "import re\n",
    "import requests\n",
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "from lxml import etree\n",
    "#url ='https://book.qidian.com/info/1015207338#Catalog'\n",
    "#请求头\n",
    "#爬取多本小说\n",
    "global clist\n",
    "global ann\n",
    "#url ='https://book.qidian.com/info/1015207338#Catalog'\n",
    "header={'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}\n",
    "url1='https://www.qidian.com/all?orderId=&page=1&vip=0&style=1&pageSize=20&siteid=1&pubflag=0&hiddenField=0'\n",
    "#获取小说名以及小说的链接\n",
    "def getbooklist(url1):\n",
    "    books=requests.get(url1,headers=header).text\n",
    "    objects=etree.HTML(books)\n",
    "    #<div class=\"book-mid-info\">\n",
    "    objs=objects.xpath('//div[@class=\"book-mid-info\"]/h4')\n",
    "    ann=[]\n",
    "    for obj in objs:\n",
    "        try:\n",
    "            book_urls=obj.xpath('a/@href')[0]\n",
    "            print(book_urls)\n",
    "        #print(obj)\n",
    "        #获取小说名称,不属于属性\n",
    "            book_names=obj.xpath('a/text()')[0]\n",
    "            print(book_names)\n",
    "        #字典\n",
    "            into1 ={\n",
    "                  'book_urls':'https:'+book_urls,\n",
    "                  'book_names':book_names\n",
    "            }\n",
    "            ann.append(into1)\n",
    "        except:\n",
    "            pass\n",
    "    #clist=[]\n",
    "    #clist.append(into)\n",
    "    return ann\n",
    "    #text=' '\n",
    "    #for j in content1:\n",
    "        #text=text+j\n",
    "#global clist\n",
    "#url ='https://book.qidian.com/info/1015207338#Catalog'\n",
    "#header={'user-agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'}\n",
    "#获取章节名以及章节的链接\n",
    "def getbookurls(url):\n",
    "    #获取页面源代码\n",
    "    charptes=requests.get(url,headers=header).text\n",
    "    objects=etree.HTML(charptes)\n",
    "    #//匹配所有\n",
    "    objs=objects.xpath('//ul[@class=\"cf\"]/li')\n",
    "    #print(objs)\n",
    "    #用try,except检查错误\n",
    "    clist=[]\n",
    "    for obj in objs:\n",
    "        try:\n",
    "            chapt_urls=obj.xpath('a/@href')[0]\n",
    "            print(chapt_urls)\n",
    "        #print(obj)\n",
    "        #获取小说名称,不属于属性\n",
    "            chapt_names=obj.xpath('a/text()')[0]\n",
    "            print(chapt_names)\n",
    "        #字典\n",
    "            into ={\n",
    "                  'chapt_urls':'https:'+chapt_urls,\n",
    "                  'chapt_names':chapt_names\n",
    "            }\n",
    "            clist.append(into)\n",
    "        except:\n",
    "            pass\n",
    "    #clist=[]\n",
    "    #clist.append(into)\n",
    "    return clist\n",
    "#clist=getbookurls(url)\n",
    "#print(clist)\n",
    "#获取小说具体内容\n",
    "def getcontent(url):\n",
    "    #获取数据\n",
    "    res=requests.get(url,headers=header).text\n",
    "    objects=etree.HTML(res)\n",
    "    #解析内容\n",
    "    objs=objects.xpath('//div[@class=\"read-content j_readContent\"]/p/text()')\n",
    "    content=[]\n",
    "    for i in objs:\n",
    "        print(i)\n",
    "        text=i.replace('\\u3000\\u3000',' ')\n",
    "        content.append(text)\n",
    "    return content\n",
    "\n",
    "    #clist=getbookurls()\n",
    "ann=getbooklist(url1)\n",
    "num=0\n",
    "for i in ann:\n",
    "    book_urls=i['book_urls']\n",
    "    book_name=i['book_names']\n",
    "    print(\"正在下载 %s\"%book_name)\n",
    "    clist=getbookurls(book_urls)\n",
    "    num=num+1\n",
    "    #(school_name+'.data').decode('utf-8')\n",
    "    #with open(r'E:\\python\\小说'+str(num)+'\\%s'%book_name , 'w')as f:\n",
    "       # f.write(book_name)\n",
    "    path=\"E:\\\\python\\\\noval\"+str(num)+str(book_name)\n",
    "    #新建文件夹\n",
    "    if not os.path.exists(path):\n",
    "    #path=\"E:\\\\python\\\\noval\"+str(num)+str(book_name)\n",
    "            os.mkdir(path, 755)\n",
    "            #print(\"正在下载 %s\"%book_name)\n",
    "            for i in clist:\n",
    "                chapt_urls=i['chapt_urls']\n",
    "                chapt_name=i['chapt_names']\n",
    "                content=getcontent(chapt_urls)\n",
    "                text=' '\n",
    "                for j in content:\n",
    "                    text=text+j\n",
    "                    #写入文件\n",
    "                    with open(r'E:\\\\python\\\\noval'+str(num)+str(book_name)+'\\\\%s'%chapt_name,'w')as f:\n",
    "                        try:\n",
    "                            f.write(text)\n",
    "                        except:\n",
    "                            pass\n",
    "    else:\n",
    "        pass\n",
    "    #text=' '\n",
    "    #for j in content1:\n",
    "        #text=text+j\n",
    "    #print(\"正在下载 %s\"%book_name)\n",
    "    time_sleep = 100\n",
    "    #print(\"正在下载 %s\"%chapt_name)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 词频分析，关键词抽取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import jieba\n",
    "import os\n",
    "from jieba import analyse\n",
    "# 引入TF-IDF关键词抽取接口\n",
    "tfidf = analyse.extract_tags\n",
    "rootdir0 = r'E:\\python'\n",
    "list0 = os.listdir(rootdir0) #列出文件夹下所有的目录与文件\n",
    "print(list0)\n",
    "for j in range(0,len(list0)):\n",
    "    data={}\n",
    "    path = os.path.join(rootdir0,list0[j])\n",
    "    if os.path.isdir(path):\n",
    "        print(path)\n",
    "        li = os.listdir(path) \n",
    "        for i in range(0,len(li)):\n",
    "            path1 = os.path.join(path,li[i])\n",
    "            if os.path.isfile(path1):\n",
    "                fb=open(path1,'r')\n",
    "                content=fb.read()\n",
    "                fb.close()\n",
    "        #words=jieba.lcut(content)\n",
    "                keywords = tfidf(content)\n",
    "                for word in keywords:\n",
    "                    if len(word)>1:#去掉单个字符\n",
    "                        if word in data:\n",
    "                            data[word]+=1\n",
    "                        else:\n",
    "                            data[word]=1\n",
    "        hist= list(data.items())#转成列表，便于排序\n",
    "        hist.sort(key=lambda x:x[1],reverse=True)\n",
    "#排序\n",
    "        for i in range(50):\n",
    "            print('{:<10}{:>5}'.format(hist[i][0],hist[i][1]))\n",
    "#print(hist)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 情感分析"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#coding:UTF-8\n",
    "import jieba\n",
    "import sys\n",
    "import os\n",
    "from snownlp import SnowNLP\n",
    "from snownlp import sentiment\n",
    "input_file=r'E:\\python\\noval1从1983开始'\n",
    "#output_file='E:/result.txt/'\n",
    "def read_and_analysis(input_file):\n",
    "    #f = open(r'E:\\python\\noval1从1983开始)         \n",
    "    rootdir = r'E:\\python\\noval1从1983开始'\n",
    "    list = os.listdir(rootdir) #列出文件夹下所有的目录与文件\n",
    "    #print(list)\n",
    "    for j in range(0,len(list)):\n",
    "    #data={}\n",
    "        path = os.path.join(rootdir,list[j])\n",
    "        #print(path)\n",
    "        if os.path.isfile(path):\n",
    "            #print(path)\n",
    "            fb=open(path,'r')\n",
    "            while True:\n",
    "                line = fb.readline()\n",
    "                #print(line)\n",
    "                if not line:\n",
    "                        break\n",
    "                lines = line.strip().split(\"\\t\")\n",
    "                #print(lines)\n",
    "                #if len(lines) < 2:\n",
    "                   #print(len(lines))\n",
    "                    #continue\n",
    "                s= SnowNLP(lines[0])\n",
    "    # s.words 查询分词结果\n",
    "                seg_words = \"\"\n",
    "        \n",
    "                print(s.keywords(50))\n",
    "                for x in s.words:\n",
    "                    seg_words += \"_\"\n",
    "                    seg_words += x\n",
    "                print(s.sentiments)\n",
    "    print(s.sentiments)\n",
    "    fw.close()\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # 重新训练模型\n",
    "    sentiment.train('E:/result/neg.txt', 'E:/result/pos.txt')\n",
    "  # 保存好新训练的模型\n",
    "    sentiment.save('sentiment.marshal')\n",
    "    read_and_analysis(input_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#coding:utf-8\n",
    "require 'rest-client'\n",
    "require 'json'\n",
    "\n",
    "def send_file(*argv)\n",
    "    link = 'http://api.bosonnlp.com/sentiment/analysis'\n",
    "    # 填写注册得到的API Token\n",
    "    headers = {\n",
    "        'X-Token'=>'xxxxx',\n",
    "        'Content-Type'=>'application/json',\n",
    "        'Accept'=>'application/json'\n",
    "    }\n",
    "    a =[]\n",
    "     argv.each do |x|\n",
    "        s = File.read(x)\n",
    "        data = s.to_json\n",
    "        resp = RestClient.post(link,data,headers)\n",
    "        v = resp.gsub(\"[\",\"\").gsub(\"]\",\"\").split(',')\n",
    "        res = {\"file_name\"=>x,\"optimism\"=>v[0].strip,\"pessimistic\"=>v[1].strip}\n",
    "        a<< res\n",
    "     end\n",
    "     puts a\n",
    "end\n",
    "send_file(\"t.txt\",\"1.txt\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}