Python爬蟲爬取愛奇藝電影片庫首頁的實(shí)例代碼

import time
import traceback
import requests
from lxml import etree
import re
from bs4 import BeautifulSoup
from lxml.html.diff import end_tag
import json
import pymysql
#連接數(shù)據(jù)庫  獲取游標(biāo)
def get_conn():
    """
    :return: 連接，游標(biāo)
    """
    # 創(chuàng)建連接
    conn = pymysql.connect(host="82.157.112.34",
                    user="root",
                    password="root",
                    db="MovieRankings",
                    charset="utf8")
    # 創(chuàng)建游標(biāo)
    cursor = conn.cursor()  # 執(zhí)行完畢返回的結(jié)果集默認(rèn)以元組顯示
    if ((conn != None)  (cursor != None)):
        print("數(shù)據(jù)庫連接成功！游標(biāo)創(chuàng)建成功！")
    else:
        print("數(shù)據(jù)庫連接失?。?)
    return conn, cursor
#關(guān)閉數(shù)據(jù)庫連接和游標(biāo)
def close_conn(conn, cursor):
    if cursor:
        cursor.close()
    if conn:
        conn.close()
    return 1
def get_iqy():
    #   獲取數(shù)據(jù)庫總數(shù)據(jù)條數(shù)
    conn, cursor = get_conn()
    sql = "select count(*) from movieiqy"
    cursor.execute(sql)     #   執(zhí)行sql語句
    conn.commit()       #   提交事務(wù)
    all_num = cursor.fetchall()[0][0]       #cursor 返回值的類型是一個(gè)元祖的嵌套形式 比如( ( ) ,)
    pagenum=int(all_num/48)+1               #這里是計(jì)算一個(gè)下面循環(huán)的起始值    每48個(gè)電影分一組
    print(pagenum)
    print("movieiqy數(shù)據(jù)庫有", all_num, "條數(shù)據(jù)！")


    url = "https://pcw-api.iqiyi.com/search/recommend/list?channel_id=1data_type=1mode=11page_id=1ret_num=48session=ee4d98ebb4e8e44c8d4b14fa90615fb7"
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36"
    }
    # response=requests.get(url=url,headers=headers)
    # response.encoding="utf-8"
    # page_text=response.text
    # print(page_text)
    """
    """
    #
    temp_list = []      #暫時(shí)存放單部電影的數(shù)據(jù)
    dataRes = []        #每次循環(huán)把單部電影數(shù)據(jù)放到這個(gè)list
    for i in range(pagenum+1, pagenum+100):         #循環(huán)100-1次
        url = "https://pcw-api.iqiyi.com/search/recommend/list?channel_id=1data_type=1mode=11page_id=1ret_num=48session=ee4d98ebb4e8e44c8d4b14fa90615fb7"
        url_0 = "https://pcw-api.iqiyi.com/search/recommend/list?channel_id=1data_type=1mode=11page_id="
        url_0 = url_0 + str(i) + "ret_num=48session=ad1d98bb953b7e5852ff097c088d66f2"
        print(url_0)        #輸出拼接好的url
        response = requests.get(url=url_0, headers=headers)
        response.encoding = "utf-8"
        page_text = response.text
        #解析json對(duì)象
        json_obj = json.loads(page_text)
        #這里的異常捕獲是因?yàn)?    測(cè)試循環(huán)的次數(shù)有可能超過電影網(wǎng)站提供的電影數(shù) 為了防止后續(xù)爬到空的json對(duì)象報(bào)錯(cuò)
        try:
            json_list = json_obj['data']['list']
        except KeyError:
            return dataRes          #json為空 程序結(jié)束
        for j in json_list:         #   開始循環(huán)遍歷json串
            # print(json_list)
            name = j['name']        #找到電影名
            print(name)
            temp_list.append(name)
            #異常捕獲，防止出現(xiàn)電影沒有評(píng)分的現(xiàn)象
            try:
                score = j['score']      #找到電影評(píng)分
                print(score)
                temp_list.append(score)
            except KeyError:
                print( "KeyError")
                temp_list.append("iqy暫無評(píng)分")            #替換字符串

            link = j['playUrl']             #找到電影鏈接
            temp_list.append(link)
            # 解析播放狀態(tài)
            state = []
            pay_text = j['payMarkUrl']          #因?yàn)椴シ艩顟B(tài)只有在一個(gè)圖片鏈接里有 所以需要使用re解析出類似vip和only（獨(dú)播）的字樣
            if (len(pay_text) == 0):            #如果沒有這個(gè)圖片鏈接 說明電影是免費(fèi)播放
                state="免費(fèi)"
            else:
                find_state = re.compile("(.*?).png")
                state = re.findall(find_state, pay_text)        #正則匹配鏈接找到vip
                if(len(state)!=0):              #只有當(dāng)鏈接不為空再執(zhí)行
                    # print(state)
                    # 再次解析
                    state = state[0][0:3]       #字符串分片

                    # 這里只輸出了三個(gè)字符，如果是獨(dú)播，頁面顯示的是only，我們?cè)O(shè)置為”獨(dú)播“
                    if (state == "onl"):
                        state = "獨(dú)播"
                    else:
                        state = "VIP"
            # print(state)
            # 添加播放狀態(tài)
            temp_list.append(state)
            dataRes.append(temp_list)
            # print(temp_list)
            temp_list = []

        print('___________________________')
    return dataRes

def insert_iqy():
    cursor = None
    conn = None
    try:
        count=0
        list = get_iqy()
        print(f"{time.asctime()}開始插入愛奇藝電影數(shù)據(jù)")
        conn, cursor = get_conn()
        sql = "insert into movieiqy (id,name,score,path,state) values(%s,%s,%s,%s,%s)"
        for item in list:
            print(item)
            count = count + 1
            if (count % 48 == 0):
                print('___________________________')
            #異常捕獲，防止數(shù)據(jù)庫主鍵沖突
            try:
                cursor.execute(sql, [0, item[0], item[1], item[2], item[3] ])
            except pymysql.err.IntegrityError:
                print("重復(fù)！跳過！")

        conn.commit()  # 提交事務(wù) update delete insert操作
        print(f"{time.asctime()}插入愛奇藝電影數(shù)據(jù)完畢")
    except:
        traceback.print_exc()
    finally:
        close_conn(conn, cursor)
    return;

if __name__ == '__main__':
    # get_iqy()
    insert_iqy()