python爬蟲beautifulsoup庫(kù)使用操作教程全解(python爬蟲基礎(chǔ)入門)

!DOCTYPE html>
html>
head>
 meta content="text/html;charset=utf-8" http-equiv="content-type" />
 meta content="IE=Edge" http-equiv="X-UA-Compatible" />
 meta content="always" name="referrer" />
 link
href="https://ss1.bdstatic.com/5eN1bjq8AAUYm2zgoY3K/r/www/cache/bdorz/baidu.min.
css" rel="stylesheet" type="text/css" />
 title>百度一下，你就知道 /title>
/head>
body link="#0000cc">
 div >
 div >
 div >
  div >
  a  rel="external nofollow" rel="external nofollow" name="tj_trnews">新聞
/a>
  a  rel="external nofollow" 
name="tj_trhao123">hao123 /a>
  a  rel="external nofollow" name="tj_trmap">地圖 /a>
  a  rel="external nofollow" name="tj_trvideo">視頻 /a>
  a  rel="external nofollow" name="tj_trtieba">貼吧
/a>
  a  rel="external nofollow" name="tj_briicon"
>更多產(chǎn)品 /a>
  /div>
 /div>
 /div>
 /div>
/body>
/html>

import requests
from bs4 import BeautifulSoup

# 使用Requests庫(kù)加載頁(yè)面代碼
r = requests.get('https://www.baidu.com')
r.raise_for_status()  # 狀態(tài)碼返回
r.encoding = r.apparent_encoding
demo = r.text

# 使用BeautifulSoup庫(kù)解析代碼
soup = BeautifulSoup(demo,'html.parser')  # 使用html的解析器

print(soup.prettify())   # prettify 方式輸出頁(yè)面

p > ... /p> 

from bs4 import BeautifulSoup
html = 'https://www.baidu.com'
bs = BeautifulSoup(html,"html.parser")

print(bs.title) # 獲取title標(biāo)簽的所有內(nèi)容
print(bs.head) # 獲取head標(biāo)簽的所有內(nèi)容
print(bs.a)  # 獲取第一個(gè)a標(biāo)簽的所有內(nèi)容
print(type(bs.a))	# 類型

print(bs.name)
print(bs.head.name)			# head 之外對(duì)于其他內(nèi)部標(biāo)簽，輸出的值便為標(biāo)簽本身的名稱
print(bs.a.attrs) 			# 把 a 標(biāo)簽的所有屬性打印輸出了出來(lái)，得到的類型是一個(gè)字典。
print(bs.a['class']) 		# 等價(jià) bs.a.get('class') 也可以使用get方法，傳入屬性的名稱，二者是等價(jià)的
bs.a['class'] = "newClass" # 對(duì)這些屬性和內(nèi)容進(jìn)行修改
print(bs.a)
del bs.a['class']			# 對(duì)這個(gè)屬性進(jìn)行刪除
print(bs.a)

from bs4 import BeautifulSoup
html = 'https://www.baidu.com'
bs = BeautifulSoup(html,"html.parser")
print(bs.title.string)
print(type(bs.title.string))

from bs4 import BeautifulSoup
html = 'https://www.baidu.com'
bs = BeautifulSoup(html,"html.parser")
print(bs.a)
# 標(biāo)簽中的內(nèi)容a  rel="external nofollow" rel="external nofollow" name="tj_trnews">!--新聞-->/a>
print(bs.a.string) 		# 新聞
print(type(bs.a.string)) # class 'bs4.element.Comment'>

soup = BeautifulSoup(demo,'html.parser') 

# 循環(huán)遍歷兒子節(jié)點(diǎn)
for child in soup.body.children:
	print(child)

# 循環(huán)遍歷子孫節(jié)點(diǎn) 
for child in soup.body.descendants:
 print(child)
 
# 輸出子節(jié)點(diǎn)的列表形式
print(soup.head.contents)
print(soup.head.contents[1])	# 用列表索引來(lái)獲取它的某一個(gè)元素

soup = BeautifulSoup(demo,'html.parser') 

for parent in soup.a.parents:
	if parent is None:
		parent(parent)
	else:
		print(parent.name)

for sibling in soup.a.next_sibling:
	print(sibling)		# 遍歷后續(xù)節(jié)點(diǎn)
	
for sibling in soup.a.previous_sibling:
	print(sibling)		# 遍歷

a_list = bs.find_all("a")
print(a_list)		# 將會(huì)返回所有包含a標(biāo)簽的內(nèi)容

from bs4 import BeautifulSoup
import re

html = 'https://www.baidu.com'
bs = BeautifulSoup(html,"html.parser")
t_list = bs.find_all(re.compile("a"))
for item in t_list:
 	print(item)		# 輸出列表

t_list = bs.find_all(["meta","link"])
for item in t_list:
	print(item)

from bs4 import BeautifulSoup

html = 'https://www.baidu.com'
bs = BeautifulSoup(html,"html.parser")
def name_is_exists(tag):
 	 return tag.has_attr("name")
t_list = bs.find_all(name_is_exists)
for item in t_list:
 	 print(item)

t_list = bs.find_all(data-foo="value")

from bs4 import BeautifulSoup
import re

html = 'https://www.baidu.com'
bs = BeautifulSoup(html, "html.parser")
t_list = bs.find_all(attrs={"data-foo": "value"})
for item in t_list:
 	print(item)
t_list = bs.find_all(text="hao123")
for item in t_list:
 	print(item)
t_list = bs.find_all(text=["hao123", "地圖", "貼吧"])
for item in t_list:
 	print(item)
t_list = bs.find_all(text=re.compile("\d"))
for item in t_list:
 	print(item)

soup.find_all(sring = re.compile('pyhton'))		# 指定查找內(nèi)容

# 或者指定使用正則表達(dá)式要搜索的內(nèi)容
sring = re.compile('pyhton')		# 字符為python
soup.find_all(string)				# 調(diào)用方法模板