Scrapy 爬取内容 并且翻页

需求：爬取每一个问政标题的内容 并且翻页
解决方案：通过Scrapy来实现需求

问政url:
http://wz.sun0769.com/political/index/index
思路：
1.找到ul标签
2.找到li标签
3.找到li标签里面的href title
4.翻页
http://wz.sun0769.com/political/index/politicsNewest 第一页
http://wz.sun0769.com/political/index/politicsNewest?id=1&page=2 第二页
http://wz.sun0769.com/political/index/politicsNewest?id=1&page=3 第三页

yg.py

import scrapy
from yangguang.items import YangguangItem


class YgSpider(scrapy.Spider):
    name = 'yg'
    allowed_domains = ['wz.sun0769.com']
    start_urls = ['http://wz.sun0769.com/political/index/politicsNewest?id=1']

    def parse(self, response):

        li_list = response.xpath("//ul[@class='title-state-ul']/li")
        for li in li_list:

            item = YangguangItem()
            item['title'] = li.xpath('./span[3]/a/text()').extract_first()  # 标题
            item['href'] = 'http://wz.sun0769.com' + li.xpath('./span[3]/a/@href').extract_first()  # 详情页的url
            # print(item)
            # 向详情页发起请求并获取得响应的数据
            yield scrapy.Request(
                url=item['href'],
                callback=self.parse_detail,
                meta={
    
    'item': item}
            )
        # 开始进行翻页
        next_url = 'http://wz.sun0769.com' + response.xpath('//div[@class="mr-three paging-box"]/a[2]/@href').extract_first()
        print(next_url)
        if next_url is not None:

            yield scrapy.Request(
                url=next_url,
                callback=self.parse
            )
    # http://wz.sun0769.com/political/politics/index?id=492147
    def parse_detail(self, response):

        item = response.meta.get('item')

        # 获取详情页的数据
        item['content'] = response.xpath('//div[@class="details-box"]/pre/text()').extract_first()
        print(item)
        yield item

pipelines.py

import re


class YangguangPipeline:
    def process_item(self, item, spider):
        # 处理格式问题
        item['content'] = self.pares_content(item['content'])
        print(item)
        return item

    # 定义一个方法来处理格式问题
    def pares_content(self, content):
        content = re.sub(r'\r\n', '', content)
        return content

items.py

import scrapy


class YangguangItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    title = scrapy.Field()
    href = scrapy.Field()
    content = scrapy.Field()