教你如何使用Python快速爬取需要的數(shù)據(jù)

"""例"""
from urllib import request
response = request.urlopen(r'http://bbs.pinggu.org/') 
#返回狀態(tài) 200證明訪問成功
print("返回狀態(tài)碼：  "+str(response.status))
#讀取頁面信息轉(zhuǎn)換文本并進行解碼，如果本身是UTF-8就不要，具體看頁面格式
#搜索“charset”查看編碼格式
response.read().decode('gbk')[:100]

url = r'http://bbs.pinggu.org/'
headers = {'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
           r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
           'Referer': r'http://www.lagou.com/zhaopin/Python/?labelWords=label',
           'Connection': 'keep-alive'}
req = request.Request(url, headers=headers)
page = request.urlopen(req).read()
page = page.decode('gbk')
page[:100]

import urllib.parse
url = r'https://new-api.meiqia.com/v1/throttle/web?api_name=%2Fvisit%2Finitent_id=7276v=1534848690048'
headers = {'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
           r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
           'Referer': r'http://www.lagou.com/zhaopin/Python/?labelWords=label',
           'Connection': 'keep-alive'}
values={'api_name':'/visit/init','ent_id':'7276','v':'1534848690048'}
data = urllib.parse.urlencode(values).encode(encoding='UTF8')
req = request.Request(url, data,headers=headers)
page = request.urlopen(req).read()
page = page.decode('gbk')

from urllib.request import Request, urlopen
from urllib.error import URLError, HTTPError
req = Request("http://www.jb51.net /")
try:
    response = urlopen(req)
except HTTPError as e:
    print('服務(wù)器無法滿足請求.')
    print('錯誤代碼: ', e.code)
except URLError as e:
    print('不能訪問服務(wù)器.')
    print('原因: ', e.reason)
else:
    print("OK!")
    print(response.read().decode("utf8"))

from urllib.request import Request, urlopen
from urllib.error import  URLError
req = Request("http://www.jb51.net /")
try:
    response = urlopen(req)
except URLError as e:
    if hasattr(e, '原因'):
        print('不能訪問服務(wù)器.')
        print('原因: ', e.reason)
    elif hasattr(e, '代碼'):
        print('服務(wù)器無法滿足請求.')
        print('錯誤代碼: ', e.code)
    else:
        print("OK!")
        print(response.read().decode("utf8"))

import urllib.request
 
# 私密代理授權(quán)的賬戶
user = "user_name"
# 私密代理授權(quán)的密碼
passwd = "uesr_password"
# 代理IP地址  比如可以使用百度西刺代理隨便選擇即可
proxyserver = "177.87.168.97:53281"
# 1. 構(gòu)建一個密碼管理對象，用來保存需要處理的用戶名和密碼
passwdmgr = urllib.request.HTTPPasswordMgrWithDefaultRealm()
# 2. 添加賬戶信息，第一個參數(shù)realm是與遠程服務(wù)器相關(guān)的域信息，一般沒人管它都是寫None，后面三個參數(shù)分別是 代理服務(wù)器、用戶名、密碼
passwdmgr.add_password(None, proxyserver, user, passwd)
# 3. 構(gòu)建一個代理基礎(chǔ)用戶名/密碼驗證的ProxyBasicAuthHandler處理器對象，參數(shù)是創(chuàng)建的密碼管理對象
#   注意，這里不再使用普通ProxyHandler類了
proxyauth_handler = urllib.request.ProxyBasicAuthHandler(passwdmgr)
# 4. 通過 build_opener()方法使用這些代理Handler對象，創(chuàng)建自定義opener對象，參數(shù)包括構(gòu)建的 proxy_handler 和 proxyauth_handler
opener = urllib.request.build_opener(proxyauth_handler)
# 5. 構(gòu)造Request 請求
request = urllib.request.Request("http://bbs.pinggu.org/")
# 6. 使用自定義opener發(fā)送請求
response = opener.open(request)
# 7. 打印響應(yīng)內(nèi)容
print (response.read())

pip install BeautifulSoup

# 想要抓取我們需要的東西需要進行定位，尋找到標(biāo)志
from bs4 import BeautifulSoup
soup = BeautifulSoup('meta content="all" name="robots" />',"html.parser")
tag=soup.meta

# tag的類別
type(tag)
>>> bs4.element.Tag

# tag的name屬性
tag.name
>>> 'meta'

# attributes屬性
tag.attrs
>>> {'content': 'all', 'name': 'robots'}

# BeautifulSoup屬性
type(soup)
>>> bs4.BeautifulSoup
soup.name
>>> '[document]'

# 字符串的提取
markup='b>a  rel="external nofollow"  target="_blank">房產(chǎn)/a>/b>'
soup=BeautifulSoup(markup,"lxml")
text=soup.b.string
text
>>> '房產(chǎn)'
type(text)
>>> bs4.element.NavigableString

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=2"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")

#通過頁面解析得到結(jié)構(gòu)數(shù)據(jù)進行處理
from bs4 import BeautifulSoup
soup=BeautifulSoup(html.text,"lxml")

#定位
lptable = soup.find('table',width='780')

# 解析
for i in lptable.find_all("td",width="680"):    
    title = i.b.strong.a.text
    ]
#     href = i.find('a')['href']
    date = href.split("/")[4]
    print (title,href,date)

from lxml import etree

html="""
!DOCTYPE html>
html>
head lang="en">
    title>test/title>
    meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
/head>
body>
div id="content">
    ul id="ul">
        li>NO.1/li>
        li>NO.2/li>
        li>NO.3/li>
    /ul>
    ul id="ul2">
        li>one/li>
        li>two/li>
    /ul>
/div>
div id="url">
    a  rel="external nofollow"  title="crossgate">crossgate/a>
    a  rel="external nofollow"  title="pinggu">pinggu/a>
/div>
/body>
/html>
"""

#這里使用id屬性來定位哪個div和ul被匹配 使用text()獲取文本內(nèi)容
selector=etree.HTML(html)
content=selector.xpath('//div[@id="content"]/ul[@id="ul"]/li/text()') 
for i in content:
    print (i)

#這里使用//從全文中定位符合條件的a標(biāo)簽，使用“@標(biāo)簽屬性”獲取a便簽的href屬性值
con=selector.xpath('//a/@href') 
for i in con:
    print (i)

#使用絕對路徑 #使用相對路徑定位 兩者效果是一樣的
con=selector.xpath('/html/body/div/a/@title') 
print (len(con))
print (con[0],con[1])

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=1"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text[1900:2000]

import requests
from bs4 import BeautifulSoup
url = "http://www.cwestc.com/MroeNews.aspx?gd=2"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text[1900:2000]

import requests
from bs4 import BeautifulSoup
url = "http://news.cqcoal.com/blank/nl.jsp?tid=238"
html = requests.get(url)
soup =  BeautifulSoup(html.text,"lxml")
soup.text

import urllib
import urllib.request
import requests

url = "http://news.cqcoal.com/manage/newsaction.do?method:webListPageNewsArchivesByTypeid"
post_param = {'pageNum':'1',\

              'pageSize':'20',\

              'jsonStr':'{"typeid":"238"}'}
return_data = requests.post(url,data =post_param)
content=return_data.text
content