import requests
import re
import os
from gooey import Gooey, GooeyParser
import time
s = requests.Session()
def judgeTypeOfPath(name):
'''
判斷該路徑是文件還是文件夾
:param name: 路徑名稱
:return:True->文件;False->文件夾
'''
if name[-1] == '/':
return False
else:
return True
def makeDirOfPath(path):
'''
創(chuàng)建文件夾
:param path: 文件夾名稱以及路徑
:return: True->創(chuàng)建成功;False->創(chuàng)建失敗
'''
if not os.path.isdir(path):
os.mkdir(path)
if not os.path.isdir(path):
return False
return True
def getPath(url):
'''
獲取網(wǎng)頁路徑列表
:param url: 當(dāng)前網(wǎng)頁路徑
:return: 路徑列表
'''
baseResponse = s.get(url=url, stream=True,verify=False).text
listOfDirOrFilesTemp = re.findall(r'li>a href=".*?" rel="external nofollow" >', baseResponse)
listOfDirOrFiles = []
for i in range(len(listOfDirOrFilesTemp)):
listOfDirOrFiles.append(listOfDirOrFilesTemp[i].split("\"")[1])
return listOfDirOrFiles[1:len(listOfDirOrFiles) + 1]
def rfSearch(listOfPath,url, nowPath):
'''
遞歸尋找目錄、路徑,并下載文件
:param listOfPath: 當(dāng)前目錄下文件以及文件夾目錄列表
:param nowPath: 現(xiàn)在所在路徑
:return:
'''
newList = listOfPath[:]
if not newList:
return
for i in range(len(newList)):
if not judgeTypeOfPath(newList[i]):
u = nowPath + newList[i][0:len(newList[i])]
makeDirOfPath(u)
tempPath=nowPath + newList[i][0:len(newList[i])+1]
tempUrl=url+newList[i][0:len(newList[i])+1]
u=getPath(tempUrl)
rfSearch(u,tempUrl,tempPath)
else:
print(f'開始下載{newList[i]}...')
t1=time.time()
u = nowPath + newList[i]
m=url+newList[i]
if not os.path.exists(u):
r = s.get(m, stream=True,verify=False)
f = open(u, "wb")
for chunk in r.iter_content(chunk_size=10240):
if chunk:
f.write(chunk)
f.close()
t2=time.time()
print(f'{newList[i]}下載完成\t\t用時 {t2-t1}')
@Gooey(
program_name='isric數(shù)據(jù)下載器',
encoding="utf-8", )
def main():
parser = GooeyParser(description="isric數(shù)據(jù)下載器")
parser.add_argument('--url',default=r'https://files.isric.org/soilgrids/latest/data/')
parser.add_argument('--path', widget="DirChooser", default=r'F:/isricData/')
args = parser.parse_args()
url=args.url
nowPath = args.path
u = getPath(url)
rfSearch(u, url,nowPath)
###如果不需要可視化,則不用gooey,可以將上面部分替換如下
#@Gooey(
# program_name='isric數(shù)據(jù)下載器',
# encoding="utf-8", )
#上面三行刪除即可
###main函數(shù)替換成下面部分:
# def main():
# url=r'https://files.isric.org/soilgrids/latest/data/'#在此處修改地址鏈接
# nowPath = r'F:/isricData/'#在此處修改文件保存地址
# u = getPath(url)
# rfSearch(u, url,nowPath)
if __name__ == "__main__":
main()
到此這篇關(guān)于python Requsets下載開源網(wǎng)站的代碼(帶索引 數(shù)據(jù))的文章就介紹到這了,更多相關(guān)python Requsets下載內(nèi)容請搜索腳本之家以前的文章或繼續(xù)瀏覽下面的相關(guān)文章希望大家以后多多支持腳本之家!