Python爬虫学习笔记（九）

#自己写一遍猫眼
import requests
import re
import json
from requests.exceptions import RequestException
def get_page(url):
    try:
        headers = {
    
    
            "user-agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3947.100 Safari/537.36"}
        response = requests.get(url=url,headers=headers)
        if response.status_code == 200:
            return  response.text
        return None
    except RequestException:
        return None
#  <dd>
#                         <i class="board-index board-index-11">11</i>
#     <a href="/films/78099" title="疯狂原始人" class="image-link" data-act="boarditem-click" data-val="{movieId:78099}">
#       <img src="//s3plus.meituan.net/v1/mss_e2821d7f0cfe4ac1bf9202ecf9590e67/cdn-prod/file:5788b470/image/loading_2.e3d934bf.png" alt="" class="poster-default" />
#       <img data-src="https://p1.meituan.net/movie/bc022b86345c643ca21d759166f77a553679589.jpg@160w_220h_1e_1c" alt="疯狂原始人" class="board-img" />
#     </a>
#     <div class="board-item-main">
#       <div class="board-item-content">
#               <div class="movie-item-info">
#         <p class="name"><a href="/films/78099" title="疯狂原始人" data-act="boarditem-click" data-val="{movieId:78099}">疯狂原始人</a></p>
#         <p class="star">
#                 主演：尼古拉斯·凯奇,艾玛·斯通,瑞安·雷诺兹
#         </p>
# <p class="releasetime">上映时间：2013-04-20</p>    </div>
#     <div class="movie-item-number score-num">
# <p class="score"><i class="integer">9.</i><i class="fraction">5</i></p>
#     </div>
#
#       </div>
#     </div>
#
#                 </dd>
def parse_page(resp_text):
    pattern = re.compile('<dd.*?board-index.*?>(\d+)</i>'
                         +'.*?<img data-src="(.*?)" alt="(.*?)".*?>'
                         +'.*?star">(.*?)</p>.*?'
                         +'releasetime">(.*?)</p>.*?'
                         +'integer">(.*?)</i>.*?'
                         +'fraction">(.*?)</i>',re.S)
    # pattern = re.compile('<dd>.*?board-index.*?>(\d+)</i>.*?data-src="(.*?)"'
    #                      + '.*?alt="(.*?)".*?<p class="star">(.*?)</p>'
    #                      + '.*?releasetime">(.*?)</p>.*?'
    #                      + 'integer">(.*?)</i>.*?'
    #                      + 'fraction">(.*?)</i>', re.S)
    items = re.findall(pattern,resp_text)  ###findall()返回的是元素为元组的列表
    for item in items:
        yield{
    
    
            'index':item[0],
            'image':item[1],
            'title':item[2],
            'actor':item[3].strip()[3:] if len(item[3])>3 else '',
            'time':item[4].strip()[5:] if len(item[4])>5 else '',
            'score':item[5].strip()+item[6].strip()
        }
def write_to_file(result):
    with open('catmovie2.txt','a',encoding='utf-8') as fp:
        fp.write(json.dumps(result,ensure_ascii=False)+'\n')

def main():
    #https://maoyan.com/board/4?offset=10
    url = 'https://maoyan.com/board/4'
    resp_text = get_page(url)
    results = parse_page(resp_text)
    for result in results:
        print(result)
        write_to_file(result)
if __name__=='__main__':
    main()