使用Python 爬取豆瓣热门电影

使用Python 爬取豆瓣热门电影所依赖的库

import requests
import json
import string
import urllib.parse
from urllib.parse import urlencode
from bs4 import BeautifulSoup

爬取出这些信息比如电影名称电影的评分等等废话不多说直接上代码

爬取方法

def douban(name, count):
    #  请求地址
    url = "https://movie.douban.com/j/search_subjects?type=movie&tag=%E7%83%AD%E9%97%A8&"
    #  请求头
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.62 Safari/537.36',
        'Cookie': '你的cookie信息'
    }
    # headers = {
    #     "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.132 Safari/537.36"
    # }
    # 携带的参数sort=time&page_limit=20&page_start=0
    parm = {
        'sort': 'time',
        'page_limit': 20,
        'page_start': count,
    }
    # 拼接请求参数
    ajax_url = url + urlencode(parm)
    # 发送请求
    data_json = requests.get(ajax_url, headers=headers).json()
    with open('D:\\HOME\\python\\Test02\\data\\豆瓣数据.txt', 'a') as output:
        for index, data in enumerate(data_json['subjects']):
            # 评分
            rate = data['rate']
            # 标题
            title = data['title']
            # 图片
            cover = data['cover']
            # 发送下一次请求
            data_ = requests.get(data['url'], headers=headers).content.decode()
            print("正在准备下文件写入："+title)
            # 格式转换
            soupData = BeautifulSoup(data_, 'lxml')
            #  解析数据
            aa = soupData.find(class_='subjectwrap clearfix')
            info = aa.find(attrs={'id': 'info'})
            try:
                if (len(info.find_all(class_='pl')) == 10 ):
                    #写入文件
                    output.write(ten(rate, title, cover, info) + "\n")
                    output.flush()
                if (len(info.find_all(class_='pl')) == 7):
                     # 写入文件
                     output.write(seven(rate, title, cover, info) + "\n")
                     output.flush()
                print("成功向文件写入：" + title)
            except Exception:
                print("格式解析异常：" + title)

        output.close()

解析方法

def seven(rate, title, cover, info):
    # 导演
    directors = []
    for s in info.find_all(attrs={'rel': 'v:directedBy'}):
        directors.append(s.string)
    # 主演
    protagonists = []
    for s in info.find_all(attrs={'rel': 'v:starring'}):
        protagonists.append(s.string)
    # 类型
    types = []
    for s in info.find_all(attrs={'property': 'v:genre'}):
        types.append(s.string)
    # 解析 制片国家
    ProductsCountry = info.find_all(class_='pl')[3].next_sibling
    # 语言
    language = info.find_all(class_='pl')[4].next_sibling
    # 上映日期
    date = info.find(attrs={'property': 'v:initialReleaseDate'}).string
    # 片长
    runtime = info.find(attrs={'property': 'v:runtime'}).string
    # 将数据保存到集合中
    list = {'rate': rate, 'title': title, 'cover': cover, 'directors': directors, 'protagonists': protagonists,
            'types': types, 'ProductsCountry': ProductsCountry,
            'language': language, 'date': date, 'runtime': runtime}
    # 返回集合
    return json.dumps(list)
def ten(rate, title, cover, info):
    # 导演
    directors = []
    for s in info.find_all(attrs={'rel': 'v:directedBy'}):
        directors.append(s.string)
    # 主演
    protagonists = []
    for s in info.find_all(attrs={'rel': 'v:starring'}):
        protagonists.append(s.string)
    # 类型
    types = []
    for s in info.find_all(attrs={'property': 'v:genre'}):
        types.append(s.string)
        # 解析 制片国家
    ProductsCountry = info.find_all(class_='pl')[4].next_sibling
    # 语言
    language = info.find_all(class_='pl')[5].next_sibling
    # 上映日期
    date = info.find(attrs={'property': 'v:initialReleaseDate'}).string
    # 片长
    runtime = info.find(attrs={'property': 'v:runtime'}).string
    alternateName = info.find_all(class_='pl')[8].next_sibling
    # 将数据保存到集合中
    list = {'rate': rate, 'title': title, 'cover': cover, 'directors': directors, 'protagonists': protagonists,
            'types': types, 'ProductsCountry': ProductsCountry,
            'language': language, 'date': date, 'runtime': runtime, 'alternateName': alternateName}
    # 返回集合
    return json.dumps(list)

主函数

if __name__ == '__main__':
    name = urllib.parse.quote("热门", safe=string.ascii_letters)
    for i in range(0, 10):
        douban(name=name, count=i *20)

好了整体的代码就在这里来让我们一起看一下的爬取的结果