Python爬虫学习笔记（五）

# 使用美丽汤爬取三国演义
#  定位元素和属性三种方式：beautifulsoup 正则 xpath
#  soup.tagname   soup.find('')     soup.find_all('')     soup.select('')
import requests
from bs4 import BeautifulSoup
if __name__=="__main__":
    headers = {
    
    
        "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3947.100 Safari/537.36"}
    url = 'https://www.shicimingju.com/book/sanguoyanyi.html'
    page_text = requests.get(url=url,headers=headers).text
    soup = BeautifulSoup(page_text,'lxml')
    li_list = soup.select('.book-mulu > ul > li')
    fp = open('./sanguo.txt','w',encoding='utf-8')
    for li in li_list:
        title = li.a.string  ########标签直系文本 子文本 不含孙文本
        #https://www.shicimingju.com/book/sanguoyanyi/6.html
        detail_url = 'https://www.shicimingju.com'+ li.a['href']
        detail_text = requests.get(url=detail_url,headers=headers).text
        soup  = BeautifulSoup(detail_text,'lxml')  #详情页的soup  之前的soup传的是主页面text
        content_text = soup.find('div',class_='chapter_content').text #.text get_text() 获取这个标签下的所有text元素 包括子元素的text
        #.string 只获取该标签这一级的text元素
        fp.write(title+':'+content_text+'\n')
        print(title,'爬取成功！！')