# 爬虫文件 # -*- coding: utf-8 -*- import scrapy import re from soufangwang.items import NewHouseItem,SecondhandHouseItem class FangspiderSpider(scrapy.Spider): name = 'fangSpider' allowed_domains = ['fang.com'] start_urls = ['https://www.fang.com/SoufunFamily.html'] def parse(self, response): # print(response) trs = response.xpath("//div[@class='outCont']//tr") province = None for tr in trs: #找没有class='font01' 属性的td标签 注意 tds = tr.xpath(".//td[not(@class)]") province_td = tds[0] province_text = province_td.xpath('.//text()').get() # 把空白字符'\s'去掉 province_text = re.sub(r'\s','',province_text) if province_text: province = province_text # 不爬取海外城市 if province == '其它': continue city_td = tds[1] city_links = city_td.xpath('.//a') for city_link in city_links: city = city_link.xpath('.//text()').get() city_url = city_link.xpath('.//@href').get() # 构建新房url 连接 new_url = city_url.split(".") if new_url[0].endswith('bj'): newhouse_url = "https://"+"newhouse."+"fang.com/house/s/" secondhand_url="https://"+"esf."+"fang.com/" else: newhouse_url = new_url[0]+".newhouse."+new_url[1]+"."+new_url[2]+"house/s/" # 构建二手房的url secondhand_url = new_url[0]+".esf."+new_url[1]+"."+new_url[2] print(city,'新房链接:',newhouse_url) print(city,'二手房链接:',secondhand_url) yield scrapy.Request(url=newhouse_url,callback=self.parse_newhouse,meta={'info':(province,city,newhouse_url)}) yield scrapy.Request(url=secondhand_url,callback=self.parse_esf,dont_filter=True,meta={'info':(province,city,secondhand_url)}) # break # break # 解析新房数据 def parse_newhouse(self,response): try: # 请求传参 province,city,newhouse_url = response.meta.get('info') li_list = response.xpath('//div[contains(@class,"nl_con")]/ul/li') for li in li_list: try: name = li.xpath('.//div[@class="nlcd_name"]/a/text()').get().strip() # print(name) house_type_list = li.xpath('.//div[contains(@class,"house_type")]//text()').getall() house_type_list = list(map(lambda x:x.replace("\n","").replace("\t",""),house_type_list)) house_type_str = ''.join(house_type_list).strip().split("－") house_type = house_type_str[0] area = house_type_str[1] # print(house_type) # print(area) address = li.xpath('.//div[@class="address"]/a/text()').getall() address = list(map(lambda x:x.replace("\n","").replace("\t",""),address))[1] district = li.xpath('.//div[@class="address"]/a/span/text()').get().strip() district = district[1:-1] # print(district) sale = li.xpath('.//div[@class="fangyuan"]/span[1]/text()').get() # print(sale) price = li.xpath('.//div[@class="nhouse_price"]//text()').getall() price = list(map(lambda x:x.replace("\n","").replace("\t",""),price)) price = "".join(price) # print(price) origin_url = li.xpath('.//div[@class="nlcd_name"]/a/@href').get() origin_url = "https:" + origin_url # print(origin_url) item = NewHouseItem(province=province,city=city,name=name,price=price,rooms=house_type, area=area,address=address,district=district,sale=sale,origin_url=origin_url) yield item except: pass # 下一页 next_url = response.xpath('//div[@class="page"]//a[@class="next"]/@href').get() if next_url: next_url = next_url.split("/")[-2] next_url = newhouse_url+next_url # print(next_url) yield scrapy.Request(url=next_url, callback=self.parse_newhouse,meta={'info':(province,city,newhouse_url)}) except: print("无房子信息") # 解析二手房数据 def parse_esf(self,response): try: # 请求传参 province,city,secondhand_url = response.meta.get('info') dls = response.xpath('//div[@class="main945 floatl"]/div[@class="shop_list shop_list_4"]/dl[@dataflag="bg"]') for dl in dls: name = dl.xpath('.//h4[@class="clearfix"]//span[@class="tit_shop"]/text()').get() # print(name) price = dl.xpath('.//dd[@class="price_right"]/span//text()').getall() unit = price[-1] price = list(map(lambda x:x.replace("\n","").replace("\t",""),price))[:-1] price = "".join(price) room_info = dl.xpath('.//p[@class="tel_shop"]/text()').getall() room_info = list(map(lambda x:x.replace("\n","").replace("\r","").strip(),room_info))[:5] if room_info[0] == "独栋": rooms = room_info[1] area = room_info[3] floor = room_info[0] toward = room_info[4] else: try: rooms = room_info[0] area = room_info[1] floor = room_info[2] print(province, city) print(room_info) toward = room_info[3] year = room_info[4] except: pass address = dl.xpath('.//p[@class="add_shop"]/span/text()').get() # print(address) origin_url = dl.xpath('.//h4[@class="clearfix"]/a/@href').get() origin_url = secondhand_url[:-1]+origin_url # print(origin_url) item = SecondhandHouseItem(province=province,city=city,name=name,price=price,unit=unit,rooms=rooms, area=area,floor=floor,toward=toward,year=year,address=address,origin_url=origin_url) yield item # 下一页 next_url = response.xpath('//div[@class="page_al"]/p/a/@href').get() if next_url: next_url = secondhand_url[:-1] + next_url # print(next_url) yield scrapy.Request(url=next_url, callback=self.parse_esf,meta={'info':(province,city,secondhand_url)}) except: print("本页没有房源信息")
# 管道文件 from scrapy.exporters import JsonLinesItemExporter class SoufangwangPipeline(object): def __init__(self): self.newhouse_fp = open('newhouse.json','wb') self.esfhouse_fp = open('esfhouse.json','wb') self.newhouse_exporter=JsonLinesItemExporter( self.newhouse_fp,ensure_ascii=False ) self.esfhouse_exporter=JsonLinesItemExporter( self.esfhouse_fp,ensure_ascii=False ) def process_item(self, item, spider): self.newhouse_exporter.export_item(item) self.esfhouse_exporter.export_item(item) return item def close_spider(self,spider): self.newhouse_fp.close() self.esfhouse_fp.close()
# item文件 # -*- coding: utf-8 -*- # Define here the models for your scraped items # # See documentation in: # https://doc.scrapy.org/en/latest/topics/items.html import scrapy class NewHouseItem(scrapy.Item): # define the fields for your item here like: # 省份 province = scrapy.Field() # 城市 city = scrapy.Field() # 小区名字 name = scrapy.Field() # 价格 price = scrapy.Field() # 几居室, 是个列表 rooms = scrapy.Field() # 面积 area = scrapy.Field() # 地址 address = scrapy.Field() # 行政区 district = scrapy.Field() # 是否在售 sale = scrapy.Field() # 房天下详情页面 origin_url = scrapy.Field() class SecondhandHouseItem(scrapy.Item): # define the fields for your item here like: # 省份 province = scrapy.Field() # 城市 city = scrapy.Field() # 小区名字 name = scrapy.Field() # 价格 price = scrapy.Field() # 单价 unit = scrapy.Field() # 几居室, 是个列表 rooms = scrapy.Field() # 层 floor = scrapy.Field() # 朝向 toward = scrapy.Field() # 年代 year = scrapy.Field() # 面积 area = scrapy.Field() # 地址 address = scrapy.Field() # 房天下详情页面 origin_url = scrapy.Field()
# 中间件 设置随机请求头 # -*- coding: utf-8 -*- import random class UserAgentDownloadMiddleware(object): USER_AGENTS = [ "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 " "(KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1", "Mozilla/5.0 (X11; CrOS i686 2268.111.0) AppleWebKit/536.11 " "(KHTML, like Gecko) Chrome/20.0.1132.57 Safari/536.11", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.6 " "(KHTML, like Gecko) Chrome/20.0.1092.0 Safari/536.6", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.6 " "(KHTML, like Gecko) Chrome/20.0.1090.0 Safari/536.6", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.1 " "(KHTML, like Gecko) Chrome/19.77.34.5 Safari/537.1", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/536.5 " "(KHTML, like Gecko) Chrome/19.0.1084.9 Safari/536.5", "Mozilla/5.0 (Windows NT 6.0) AppleWebKit/536.5 " "(KHTML, like Gecko) Chrome/19.0.1084.36 Safari/536.5", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1063.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1062.0 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1061.1 Safari/536.3", "Mozilla/5.0 (Windows NT 6.2) AppleWebKit/536.3 " "(KHTML, like Gecko) Chrome/19.0.1061.0 Safari/536.3", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/535.24 " "(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24", "Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/535.24 " "(KHTML, like Gecko) Chrome/19.0.1055.1 Safari/535.24" ] def process_request(self,request,spider): user_agent = random.choice(self.USER_AGENTS) request.headers['User-Agent'] = user_agent