版权声明:转载请标明出处 https://blog.csdn.net/easy_purple/article/details/82725140
参考上篇文章
附上代码:
import requests
from bs4 import BeautifulSoup
import mysql.connector
def get_pages_link():
# 插入到数据库
conn = mysql.connector.connect(user='root', password='root', database='test')
cursor = conn.cursor()
header = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36',
'Connection': 'keep-alive'
}
for item in range(0, 250, 25):
url = "https://book.douban.com/top250?start={}".format(item)
web_data = requests.get(url, headers=header)
soup = BeautifulSoup(web_data.content, 'lxml')
for movie in soup.select('.item'):
href = movie.find('a')["href"]
name = movie.get_text().strip()[:20].strip() # 片名
star = movie.select('.rating_nums')[0].text # 评分
people = movie.select('.star > span')[2].text.replace("(\n", "").replace(")", "").strip() # 评价人数
try:
quote = movie.select('.inq')[0].text
except:
print('没有quote哦')
quote = None
data = {
# 'url': href,
'评分': star,
'评价人数': people,
'片名': name,
'名言': quote
}
sql = 'insert into bookdoubantop250(score,name,quote,people) values (%f,"%s","%s","%s")' % (
float(star), name, quote, people)
cursor.execute(sql)
conn.commit()
print(data)
# print(movie)
print('\n' + '-' * 50 + '\n')
# 关闭数据库
cursor.close()
conn.close
if __name__ == '__main__':
get_pages_link()