# -*- coding:utf-8 -*-
from selenium import webdriver
from lxml import etree
import time
class BossSpider(object):
def __init__(self):
self.start_url = 'https://www.zhipin.com/job_detail/?query=python&city=101020100&industry=&position='
self.driver = webdriver.Chrome()
def get_page_url(self,source):
html = etree.HTML(source)
position_href_links = html.xpath("//div[@class='job-list']/ul/li//div[@class='info-primary']/h3/a/@href")
for href in position_href_links:
position_url = "https://www.zhipin.com" + href
print(position_url)
self.get_detail_info(position_url)
time.sleep(2)
time.sleep(2)
def get_detail_info(self,url):
self.driver.execute_script('window.open("%s")'%url)
self.driver.switch_to.window(self.driver.window_handles[1])
source = self.driver.page_source
self.parse_detail_page(source)
self.driver.close()
self.driver.switch_to.window(self.driver.window_handles[0])
def parse_detail_page(self,source):
html = etree.HTML(source)
position_info = {}
position_info["company"] = html.xpath("//div[@class='company-info']/a[2]/text()")[0].strip()
# position_info["salary"] = html.xpath("//div[@class='info-primary']/div[@class='name']/span/text()]")[0].strip()
position_info["salary"] = html.xpath("//span[@class='salary']/text()")[0].strip()
position_info["city"] = html.xpath("//div[@class='info-primary']/p//text()")[0]
position_info["work_years"] = html.xpath("//div[@class='info-primary']/p//text()")[1]
position_info["education"] = html.xpath("//div[@class='info-primary']/p//text()")[2]
position_info["desc"] = "".join(html.xpath("//div[@class='job-sec']/div/text()")).strip()
print(position_info)
def run(self):
self.driver.get(self.start_url)
while True:
source =self.driver.page_source
self.get_page_url(source)
next_btn = self.driver.find_element_by_xpath("//div[@class='page']/a[last()]")
if "next disabled" in next_btn.get_attribute("class"):
pass
else:
next_btn.click()
time.sleep(2)
time.sleep(5)
if __name__ == '__main__':
BossSpider().run()
使用selenium抓取boss直聘
猜你喜欢
转载自blog.csdn.net/u014644167/article/details/89461017
今日推荐
周排行