spider的代码:
# -*- coding: utf-8 -*-
import scrapy
from AD.items import AdItem
class AdImgSpider(scrapy.Spider):
name = 'AD_img'
allowed_domains = ['car.autohome.com.cn']
start_urls = ['https://car.autohome.com.cn/pic/brand-33.html#pvareaid=2042194']
def parse(self, response):
uiboxs = response.xpath("//div[@class='uibox']")
for uibox in uiboxs:
title = uibox.xpath(".//span[@class='fn-left']/a/@title").get()
# print(title)
urls = uibox.xpath(".//ul/li/a/img/@src").getall()
# for url in urls:
# url = response.urljoin(url)
# print(url)
urls = list(map(lambda url: response.urljoin(url), urls))
item = AdItem(title=title, urls=urls)
yield item
pipelines的代码:
# -*- coding: utf-8 -*-
# Define your item pipelines here
#
# Don't forget to add your pipeline to the ITEM_PIPELINES setting
# See: https://docs.scrapy.org/en/latest/topics/item-pipeline.html
import os
from urllib import request
from scrapy.pipelines.images import ImagesPipeline
from AD import settings
class AdPipeline(object):
def __init__(self):
self.path = os.path.join(os.path.dirname(os.path.dirname(__file__)), "images")
if not os.path.exists(self.path):
os.mkdir(self.path)
def process_item(self, item, spider):
title = item["title"]
urls = item["urls"]
title_path = os.path.join(self.path, title)
if not os.path.exists(title_path):
os.mkdir(title_path)
for url in urls:
image_name = url.split("_")[-1]
request.urlretrieve(url, os.path.join(title_path, image_name))
return item