一、scrapy延时关闭

import re
import time
import uuid
from datetime import datetime
import scrapy
from scrapy.signalmanager import SignalManager
from scrapy import signals
from pydispatch import dispatcher

class TestSpider(scrapy.Spider):
    name = "test_spider"
    start_time = time.time()

    def start_requests(self):
        SignalManager(dispatcher.Any).connect(self.closed_handler, signal=signals.spider_closed)

    def closed_handler(self):
        total_scrapy_time = time.time() - self.start_time
        remain_time = self.settings['CLOSESPIDER_TIMEOUT'] - total_scrapy_time
        if remain_time > 0:
            time.sleep(remain_time-10)