python爬取晉江文學(xué)城小說(shuō)評(píng)論(情緒分析)

import requests
from bs4 import BeautifulSoup
import bs4
import re
import csv
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import jieba
import seaborn as sns
import xlrd
from xlutils.copy import copy
# 一些魔法命令，使得matplotlib畫(huà)圖時(shí)嵌入單元中而不是新開(kāi)一個(gè)窗口
%matplotlib inline
plt.rcParams['figure.figsize'] = (10.0, 8.0) # set default size of plots
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'
%load_ext autoreload
%autoreload 2
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn import metrics
from sklearn.model_selection  import train_test_split

headers = {"User-Agent": "Mozilla/5.0"}
for n in range(1,50):
    url = 'http://www.jjwxc.net/bookbase.php?fw0=0fbsj=0ycx0=0xx2=2mainview0=0sd0=0lx0=0fg0=0sortType=0isfinish=0collectiontypes=orssearchkeywords=page={}'.format(n)
    html = requests.get(url,headers=headers)
    html.encoding = html.apparent_encoding
    soup = BeautifulSoup(html.text, 'html.parser')
    for tr in soup.find_all('tbody'):
            tds=tr('td')  
            a = tr('a') 
            count=0
            id=[]
            for u in tr.find_all('a'):
                    count=count+1
                    book_url=u.get('href') # 獲取小說(shuō)主頁(yè)的url
                    p = re.compile(r'\d+')
                    book_id = p.findall(book_url)[0]  # 獲取小說(shuō)ID
                    if(count%2==0):
                        id.append(book_id)
            for n in range(0,100):
                    with open('./data/晉江排行榜【按收藏?cái)?shù)】.txt','a+',encoding='utf-8') as f:
                            print("{0}\t{1}\t{2}".format(id[n],a[n*2+1].string,a[n*2].string),file=f)  # 序號(hào) 書(shū)名 作者

# 查看收藏榜前8部小說(shuō)的ID
with open('./data/晉江排行榜【按收藏?cái)?shù)】.txt','r',encoding='utf-8',errors='ignore') as f:
    book_list = f.readlines()
    id_list = [item.split('\t')[0] for item in book_list]
print(id_list[:8])

# 查看收藏榜前8部小說(shuō)的名字
name_list = [item.split('\t')[1] for item in book_list]
print(name_list[:8])

headers = {"User-Agent": "Mozilla/5.0"}
with open('./data/晉江排行榜【按收藏?cái)?shù)】.txt','r',encoding='utf-8') as f:
            book_list = f.readlines()
            id_list = [item.split('\t')[0] for item in book_list]
for book_id in id_list:
    for page in range(1,6):
        url="http://www.jjwxc.net/comment.php?novelid={}huati=1page={}".format(book_id,page)
        html = requests.get(url,headers=headers)
        html.encoding = html.apparent_encoding
        soup = BeautifulSoup(html.text, 'html.parser')
        scores=[]
        comments=[]
        for item1 in soup.find_all('span',"coltext"):
            score=item1('span')
            scores.append(score[2].string)
        for item2 in soup.find_all('div',"readbody"):
            comment=item2('span')
            comments.append(comment[0].string)
        for i in range(0,len(comments)):
            excel = xlrd.open_workbook('./data/jjwxc1.xls')
            wb = copy(excel)
            w_sheet = wb.get_sheet(0)
            sheet = excel.sheets()[0]
            nrow = sheet.nrows # 文件行數(shù)
            w_sheet.write(nrow, 0, book_id)
            w_sheet.write(nrow, 1, comments[i])
            w_sheet.write(nrow, 2, scores[i])
            wb.save('./data/jjwxc1.xls')

# 格式轉(zhuǎn)化
ex=pd.read_excel("./data/jjwxc.xls")
ex.to_csv("./data/jjwxc.csv",encoding="gb18030")

# 加載評(píng)論
review = pd.read_csv("./data/jjwxc.csv",names=['ID','comment','score'],encoding='gb18030')

# 去重
review = review.drop_duplicates()

# 刪除評(píng)論內(nèi)容重復(fù)的行
review= review.drop_duplicates('comment')
review.shape

# 添加情緒標(biāo)簽
review['emotion'] = (review.score ==2) * 1

# 打亂順序
review = review.sample(frac=1).reset_index(drop=True)
print(review.shape)

def review_without_stop(review):
    # 打開(kāi)停用詞文件
    with open("./data/emotion_stopwords.txt","r",encoding="utf-8") as f:
        stop_word = [x.strip() for x in f.readlines()] 
    all_stop_words = set(stop_word) # 刪除停用詞中重復(fù)的項(xiàng)
    # 短評(píng)中的非中文字符替換為''
    review = re.sub("[^\u4e00-\u9fa5]",'',review)
    # 去除全角空白字符
    review = review.replace("\u3000","") 
    # 分詞
    review = jieba.cut(review)
    # 過(guò)濾一個(gè)字的詞
    review = filter(lambda x: len(x)>1,review)
    # 去除停用詞
    review = filter(lambda x: x not in all_stop_words,review)
    return ' '.join(review)

# 自定義分詞字典
jieba.load_userdict("./data/emotion_userdict.txt")
review['cut_jieba'] = review.comment.apply(review_without_stop)

# 查看一些評(píng)論
review.head()

# 好評(píng)中一些評(píng)論包含“不想”，“不喜歡”
review[(review['cut_jieba'] == '不想')  (review['emotion'] == 1)]

review[(review['cut_jieba'] == '不喜歡')  (review['emotion'] == 1)]

# 好評(píng)中出現(xiàn)的消極情緒詞，去除這些評(píng)論
def change_negtive_like(cut_text):
    word_list = cut_text.split()
    if "不喜歡" in word_list:
        for i in range(len(word_list)):
            if word_list[i] == "不喜歡":
                word_list[i] = ""
        return " ".join(word_list)
    elif "不想" in word_list:
        for i in range(len(word_list)):
            if word_list[i] == "不想":
                word_list[i] = ""
        return " ".join(word_list)
    else:
        return cut_text

review.loc[review['emotion'] == 1,'cut_jieba'] = review[review['emotion'] == 1].cut_jieba.apply(change_negtive_like)

# 一些評(píng)論內(nèi)容為空，去除這些為空的評(píng)論
review = review[~(review['cut_jieba'] == '')]
review.shape

from wordcloud import WordCloud
from imageio import imread
mask = imread("./data/cloud.jpg")
font = './data/FZSTK.TTF'
wc = WordCloud(
    font_path= font,
    max_words=2000, # 設(shè)置最大現(xiàn)實(shí)的字?jǐn)?shù)
    max_font_size=250,# 設(shè)置字體最大值
    background_color = "white",
    random_state=30,
    mask = mask)
wc.generate(''.join(review['cut_jieba'])) # 生成詞云
plt.imshow(wc)
plt.axis('off')

from wordcloud import WordCloud
from imageio import imread
mask = imread("./data/piggy.jpg")
font = './data/FZSTK.TTF'
wc1 = WordCloud(
    font_path= font,
    max_words=2000, # 設(shè)置最大現(xiàn)實(shí)的字?jǐn)?shù)
    max_font_size=300,# 設(shè)置字體最大值
    background_color = "white",
    random_state=30,
    mask = mask)
wc1.generate(''.join(review['cut_jieba'][review['emotion']==1]))
plt.imshow(wc1)
plt.axis('off')

wc1.generate(''.join(review['cut_jieba'][review['score']==-2])) # 生成詞云
plt.imshow(wc1)
plt.axis('off')

x, y = review['cut_jieba'], review['emotion']
x_train, x_test, y_train, y_test = train_test_split(x,y,test_size=0.25) 

print(x_train.shape)
print(y_train.shape)

print(x_test.shape)
print(y_test.shape)

from sklearn.feature_extraction.text import TfidfVectorizer
tfidf_vect = TfidfVectorizer(decode_error='ignore',
                             token_pattern=r"\b[^\d\W]\w+\b", # 剔除向量化結(jié)果中的數(shù)字
                             analyzer='word',
                             ngram_range=(2,4),
                            max_df = 0.8,
                            min_df = 3)
Xtrain = tfidf_vect.fit_transform(x_train)
Xtest = tfidf_vect.transform(x_test)

print(Xtrain.shape)
print(Xtest.shape)

from sklearn.naive_bayes import MultinomialNB

review_classifier = MultinomialNB()
review_classifier.fit(Xtrain,y_train)

# 對(duì)測(cè)試集的樣本進(jìn)行預(yù)測(cè)
y_pred = review_classifier.predict(Xtest)
metrics.confusion_matrix(y_test, y_pred) # 混淆矩陣

# 利用 sns 模塊查看測(cè)試值和預(yù)測(cè)值構(gòu)成的熱圖
colorMetrics = metrics.confusion_matrix(y_test, y_pred)
sns.heatmap(colorMetrics,annot=True,fmt='d')

# 分類報(bào)告
# 給出每個(gè)類的準(zhǔn)確率，召回率和F值，以及這三個(gè)參數(shù)和宏平均值
print(metrics.classification_report(y_test,y_pred)) 

print(metrics.accuracy_score(y_test,y_pred))

from sklearn.model_selection import cross_val_score
score1 = cross_val_score(review_classifier,Xtrain,y_train,cv=10,scoring="accuracy").mean()
print(score1)

from sklearn.linear_model import LogisticRegression  
LR_model = LogisticRegression(penalty='l2',max_iter=3000)  
LR_model.fit(Xtrain,y_train)

# 對(duì)測(cè)試集的樣本進(jìn)行預(yù)測(cè)
y_pred = LR_model.predict(Xtest)
metrics.confusion_matrix(y_test, y_pred) # 混淆矩陣

print(LR_model.score(Xtest,y_test))

# 給出每個(gè)類的準(zhǔn)確率，召回率和F值，以及這三個(gè)參數(shù)和宏平均值
print(metrics.classification_report(y_test,y_pred))