python爬取慕课网课程信息

基本要求：

 分析网页源码后：

import requests
from bs4 import BeautifulSoup
from bs4 import BeautifulSoup
import requests
import io
import sys
import csv
import re
sys.stdout = io.TextIOWrapper(sys.stdout.buffer,encoding='utf-8')

f = open('abc.csv','w',encoding='gb18030',newline='')
csv_writer = csv.writer(f)
csv_writer.writerow(["标题","人物" , "岗位","难度","时长","学习人数" ,"综合评分" ,"内容实用" ,"简洁易懂" ,"逻辑清晰" ,"姓名","得分","评论内容","点赞数","时间" ,"简介"])

#open every page
def get_text(url):
	res = requests.get(url)
	res.encode = 'utf-8'
	content = res.text
	doc = BeautifulSoup(content ,'lxml')
	#print(doc)
	return doc


def get_page1(doc1):
	pageinfo1 = doc1.find('div' ,{'class' : 'course-nav-row course-nav-skills clearfix'})
	url1 = pageinfo1.findAll('a')
	url2 = url1[1:]
	longPage = []
	for item in range(len(url2)):
		attr1 = url2[item].attrs['href']
		attr2 = 'https://www.imooc.com/course/list' + attr1[12:] + '&sort=pop'
		#print(attr2)
		longPage.append(get_text(attr2))
	return longPage
		

#find avaliable message
def get_page(doc2):
	pageinfo1=doc2.findAll('a',{'class':'course-card'});
	shortTitleDoc = []

	for item in range(len(pageinfo1)):
		href1 = pageinfo1[item].attrs['href']
		if href1[0:6] == '/learn':
			href2 = 'https://www.imooc.com/coursescore' + href1[6:]
			shortTitleDoc.append(get_text(href2))
		if len(shortTitleDoc) == 3:
			return shortTitleDoc
	return shortTitleDoc

def get_introduceDoc(doc4):
	pageinfo1=doc4.findAll('a',{'class':'course-card'});
	shortTitleDoc2 = []

	for item in range(len(pageinfo1)):
		href1 = pageinfo1[item].attrs['href']
		if href1[0:6] == '/learn':
			href2 = 'https://www.imooc.com/learn' + href1[6:]
			shortTitleDoc2.append(get_text(href2))
		if len(shortTitleDoc2) == 3:
			return shortTitleDoc2
	return shortTitleDoc2

def get_AjaxSourceData(doc):
	pageinfo1=doc.findAll('a',{'class':'course-card'});
	AjaxData = []

	for item in range(len(pageinfo1)):
		href1 = pageinfo1[item].attrs['href']
		if href1[0:6] == '/learn':
			href2 = 'https://www.imooc.com/course/AjaxCourseMembers?ids=' + href1[7:]
			#print(href2)
			AjaxData.append(get_text(href2))
		if len(AjaxData) == 3:
			return AjaxData
	return AjaxData
	

def get_page_text(shortTitleDoc ,doc5 ,AjaxData):
	#顶部部分
	h2 = shortTitleDoc.find('div' ,{'class':'hd clearfix'}).find('h2').text
	#print(h2)
	title = shortTitleDoc.find('div' ,{'class' : 'statics clearfix'})
	name = shortTitleDoc.find('div' ,{'class' :'teacher-info l'})
	details = shortTitleDoc.findAll('div' ,{'class' : 'static-item'})

	name1 = name.find('span' ,{'class' : 'tit'}).find('a').text
	job1 = name.find('span' ,{'class' :'job'}).text

	metaValue = []
	for item in range(4):
		meta = details[item].find('span' ,{'class' : 'meta'}).text
		metaValue.append(details[item].find('span' ,{'class' : 'meta-value'}).text)
	#print(metaValue[0])
	r = re.findall('numbers":"(.*?)"',str(AjaxData))
	metaValue[2] = r[0]

	#中间部分
	content = shortTitleDoc.find('div' ,{'class' : 'evaluation-info'})
	#a = content.find('div' ,{'class' : 'evaluation-title'}).text
	score = content.find('div' ,{'class' : 'evaluation-score'}).text
	li = content.findAll('li')
	metaValue1 = []
	for item in range(3):
		metaValue1.append(li[item].find('span').text)
		
	#评论区部分
	comment = shortTitleDoc.find('div' ,{'class' : 'evaluation-list'})
	commentLen = comment.findAll('div' ,{'class' : 'evaluation evaluate'})
	#print(len(commentLen))
	commentName = commentLen[0].find('a' ,{'class' : 'username'}).text
	commentScore = commentLen[0].find('div' ,{'class' : 'star-box'}).find('span').text
	commentContent = commentLen[0].find('p' ,{'class' : 'content'}).text
	commentPraise = commentLen[0].find('div' ,{'class' : 'info clearfix'}).find('em').text
	commentTime = commentLen[0].find('span' ,{'class' : 'time'}).text

	content1 = doc5.find('div' ,{'class' :'course-description course-wrap'}).text

	csv_writer.writerow([h2 ,name1 ,job1 ,metaValue[0] ,metaValue[1] ,metaValue[2] ,metaValue[3] ,metaValue1[0] ,metaValue1[1] ,metaValue1[2],commentName,commentScore,commentContent,commentPraise,commentTime ,content1])

doc3 = get_text('https://www.imooc.com/course/list')
longPage1 = get_page1(doc3)
for item in range(len(longPage1)):
	shortTitleDoc1 = get_page(longPage1[item])
	shortTitleDoc3 = get_introduceDoc(longPage1[item])
	AjaxData = get_AjaxSourceData(longPage1[item])
	for i in range(len(shortTitleDoc1)):
		get_page_text(shortTitleDoc1[i] ,shortTitleDoc3[i] ,AjaxData[i])
		#get_introduce(shortTitleDoc3[i])
		#print(len(shortTitleDoc1))
f.close()

　　结果保存在CSV表格当中（部分数据）