python异步爬虫+数据分析+数据可视化

python爬虫+数据分析+数据可视化

import csv
import pandas as pd
import numpy as np
import asyncio
import aiohttp
from pandas import Series, DataFrame
# import matplotlib as mpl
import matplotlib.pyplot as plt
from lxml import etree

headers = {
    
    
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) '
                  'Chrome/84.0.4147.89 Safari/537.36 SLBrowser/7.0.0.4071 SLBChan/30 '
}


async def get_page(url):
    async with aiohttp.ClientSession() as session:
        async with await session.get(url=url, headers=headers) as response:
            # text()返回字符串形式的响应数据
            # read()返回二进制形式的响应数据
            # json()返回的就是json对象
            # 注意：获取响应数据操作之前一定要使用await进行手动挂起
            page_text = await response.text()
            tree = etree.HTML(page_text)
            titles = tree.xpath('//div[@class="property-content-title"]/h3//text()')
            values = tree.xpath('//p[@class="property-price-total"]/span[1]/text()')
            layouts = tree.xpath('//div[@class="property-content-info"]/p[1]//text()')
            a = ''
            for i in layouts:
                if i != ' ':
                    a = a + i
            layout = []
            for i in range(int(len(a) / 6)):
                layout.append(''.join(list(a)[6 * i:6 * i + 6]))

            mi = tree.xpath('//div[@class="property-content-info"]/p[2]//text()')
            location = tree.xpath('//div[@class="property-content-info"]/p[3]//text()')
            high = tree.xpath('//div[@class="property-content-info"]/p[4]//text()')
            build_times = tree.xpath('//div[@class="property-content-info"]/p[5]//text()')
            address = tree.xpath('//div[@class="property-content-info property-content-info-comm"]/p[1]//text()')
            specific_address = tree.xpath(
                '//div[@class="property-content-info property-content-info-comm"]/p[2]//text()')
            insertion = []
            for i in range(int(len(specific_address))):
                insertion.append(specific_address[i])
                if (i + 1) % 3 != 0:
                    insertion.insert(len(specific_address), '-')
            # print(insertion)
            name = tree.xpath('//div[@class="property-extra"]/span[1]/text()')
            grade = tree.xpath('//div[@class="property-extra"]/span[2]/text()')
            website = tree.xpath('//div[@class="property-extra"]/span[3]/text()')
            urls = tree.xpath('//div[@class="property"]/a[1]/@href')
            # for url in urls:
            #     print(url)
            #     new_page_text = requests.get(url=url, headers=headers, proxies={'HTTP': 'HTTP://121.230.210.132:3256'}).text
            #     new_tree = etree.HTML(new_page_text)
            #     add_time = new_tree.xpath('//div[@class="houseInfo"]/table/tbody/tr[6]/td[2]/span[2]/text()')
            #     print(add_time)
            new_specific_address = []
            for i in range(int(len(insertion) / 5)):
                new_specific_address.append(''.join(insertion[5 * i:5 * i + 5]))
            # print(new_specific_address)
            print(len(build_times))
            print(len(titles))

            for i in range(len(titles) - 1):
                new_data = [titles[i], values[i] + '万', layout[i], mi[i], location[i], high[i], build_times[i],
                            address[i],
                            new_specific_address[i], name[i], grade[i], website[i], urls[i]]
                writer.writerow(new_data)
                # fp.write('房子描述：' + titles[i] + '   ￥' + '价格：' + values[i] + '万' + '房子构造：' + layout[i] + '房子面积：' + mi[
                #     i] + '房子朝向：' + location[i] + '楼房层数：' + high[i] + '建造时间：' + build_times[i] +
                #          '地址：' + address[i] + '详细地址：' + new_specific_address[i] + '户主姓名：' + name[i] + '评分：' + grade[
                #              i] + '发布公司：' + website[i] + '网站地址：' + urls[i] + '\n')


async def main():
    urls = []

    url = 'https://bj.58.com/ershoufang/p%d/'
    for pageNum in range(1, 9):
        urls.append(format(url % pageNum))

    tasks = []

    for url in urls:
        c = get_page(url)
        task = asyncio.create_task(c)
        tasks.append(task)

    await asyncio.wait(tasks)


pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)

plt.rcParams["font.sans-serif"] = [u"SimHei"]
plt.rcParams["axes.unicode_minus"] = False

data = pd.read_csv('room01.csv', encoding='gbk')

print(data.shape)
print(data.dtypes)
print(data.columns)
# 将多余的行删除
index01 = data[data["建造时间"].str[29:33] == ''].index
data.drop(index01, inplace=True)
# 将房子面积转变为double类型新加一列mi
data['mi'] = data["房子面积"].str[29:-26].astype('double')
# 将价格转变为double类型新加一列price
data['price'] = data["￥价格"].str[:-1].astype('double')
# 将建造时间转变为int类型新加一列year
data['year'] = data["建造时间"].str[29:33].astype('int')
# print(data["建造时间"].str[29:33])
# print(data.dtypes)

# 添加一列months表示使用多少个月
data['months'] = (2021 - data['year']) * 12 + 6
# 删除评分中无用的数据
index02 = data[data['评分'].str[3:4] != '分'].index
data.drop(index02, inplace=True)
# 将评分转变为double类型新加一列grade
data['grade'] = data['评分'].str[:-1].astype('double')


def plot01():
    # 将价格分组
    price_cut = pd.cut(data['price'],
                       bins=[data['price'].min(), 100, 200, 300, 400, 500, 600, 700, 800, 900, 1000,
                             data['price'].max()])
    # print(price_cut)

    # 计算不同价格的房子数量
    price_count = price_cut.value_counts()
    # print(price_count)

    # 查看占比情况
    for i in price_count / price_count.sum():
        print(i)

    # 打印price_count索引
    print(price_count.index)

    # 绘制不同价格区间占比情况的柱状图
    X = np.arange(len(price_count))
    print(X)
    Y = price_count
    print(Y)
    plt.figure(figsize=(8, 6))
    plt.bar(X, Y, color='b', alpha=0.5)
    plt.title("二手房价格分布图")
    plt.xlabel("价格区间")
    plt.ylabel("数量")
    plt.xticks(np.arange(len(price_count)), price_count.index, rotation=30)
    plt.ylim([0, price_count.max() + 100])

    percents = [str(round(i * 100, 2)) + '%' for i in price_count / price_count.sum()]
    for x, y, z in zip(X, Y, percents):
        plt.text(x - 0.3, y + 5, z)
    plt.show()


def plot02():
    # 平均面积集合
    means = [int(data[(data['price'] < 100) & (data['price'] >= data['price'].min())]['mi'].mean()),
             int(data[(data['price'] < 200) & (data['price'] >= 100)]['mi'].mean()),
             int(data[(data['price'] < 300) & (data['price'] >= 200)]['mi'].mean()),
             int(data[(data['price'] < 400) & (data['price'] >= 300)]['mi'].mean()),
             int(data[(data['price'] < 500) & (data['price'] >= 400)]['mi'].mean()),
             int(data[(data['price'] < 600) & (data['price'] >= 500)]['mi'].mean()),
             int(data[(data['price'] < 700) & (data['price'] >= 600)]['mi'].mean()),
             int(data[(data['price'] < 800) & (data['price'] >= 700)]['mi'].mean()),
             int(data[(data['price'] < 900) & (data['price'] >= 800)]['mi'].mean()),
             int(data[(data['price'] < 1000) & (data['price'] >= 900)]['mi'].mean()),
             int(data[(data['price'] < data['price'].max()) & (data['price'] >= 1000)]['mi'].mean())]

    x = [f"[{
      
      data['price'].min()},100)", "[100,200)", "[200,300)", "[300,400)", "[400,500)", "[500,600)", "[600,700)",
         "[700,800)", "[800,900)", "[900,1000)", f"[1000,{
      
      data['price'].max()})"]

    X = np.arange(len(x))

    Y = means

    plt.figure(figsize=(8, 10))
    plt.plot(X, Y, '-..', color='b')
    plt.title('房子价格和面积之间的关系')
    plt.xlabel('价格区间')
    plt.ylabel('平均面积')
    plt.xticks(np.arange(len(X)), x, rotation=30)
    ax = plt.gca()
    for i, j in zip(X, Y):
        ax.text(i + 0.2, j + 4, j, bbox=dict(facecolor='red', alpha=0.3))
    plt.grid(True)

    plt.show()


def plot03():
    # 分析房子使用时长、面积及价格之间的关系
    plt.figure(figsize=(10, 8))

    plt.scatter(data['mi'], data['months'], s=data['price'] / 10, c='r')
    plt.xlabel("面积")
    plt.ylabel("使用月份")
    plt.show()


def plot04():
    # 分析房子评分、面积及价格之间的关系
    plt.figure(figsize=(10, 8))

    plt.scatter(data['mi'], data['grade'], s=data['price'] / 10, c='r')
    plt.xlabel("面积")
    plt.ylabel("评分")
    plt.show()


if __name__ == '__main__':
    head = ['房子描述', '￥价格', '房子构造', '房子面积', '房子朝向', '楼房层数', '建造时间', '地址', '详细地址', '户主姓名', '评分', '发布公司', '网站地址']

    with open('room02.csv', 'a', encoding='gbk', newline='')as f:
        writer = csv.writer(f)
        writer.writerow(head)
        loop = asyncio.get_event_loop()
        loop.run_until_complete(main())
        f.close()
    plot04()