python批量下载网页的方法


import urllib.request
import ssl
ssl._create_default_https_context = ssl._create_unverified_context #取消验证,用于绕过https
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.86 Safari/537.36"}# 设置头部用于绕过反爬
for i in (1,43):#假如一本书有1~42章
    print(i)
    for j in range(1,5):#假如每一章有四个小节
        url = 'https://learnXXXX.com/lesson-'+str(i)+'.html/'+str(j)
        req = urllib.request.Request(url,headers=headers)
        webContent = urllib.request.urlopen(req).read()
        f = open('lesson-'+str(i)+'-'+str(j)+'.html', 'wb')
        f.write(webContent)
        print(str(i)+'-'+str(j)+'finish')
        f.close()

发布了77 篇原创文章 · 获赞 33 · 访问量 7万+

猜你喜欢

转载自blog.csdn.net/fjh1997/article/details/104158979
今日推荐