#coding=utf-8

__author__ = 'scaleworld'

importrequestsimportreimportsys

reload(sys)

sys.setdefaultencoding("utf-8")classSpider:def __init__(self):print '开始爬取极客学院课程信息。。。'

#获取源代码

defgetSource(self, url):

html=requests.get(url)returnhtml.text#获取每个课程块信息

defgetLessons(self, source):

lessons= re.findall('deg="0" >(.*?)', source, re.S)returnlessons#获取课程信息，如课程名称、课程介绍、课程时间、课程等级、学习人数

defgetLessonInfo(self, lesson):

info={}

info['title'] = re.search('

(.*?)

', lesson, re.S).group(2).strip()

info['desc'] = re.search('

(.*?)

', lesson, re.S).group(1).strip()

timeandlevel= re.findall('(.*?)', lesson, re.S)

info['time'] = timeandlevel[0].strip().replace("\n", "").replace(" ", "")

info['level'] = timeandlevel[1].strip()

info['learnNumber'] = re.search('"learn-number">(.*?)', lesson, re.S).group(1).strip()returninfo#保存课程信息到文件LessionInfos.txt

defsaveLessionInfos(self, lessonInfos):#'w'：只写，会覆盖之前写入的内容

#也可以用'a'：追加到文件末尾

#如果文件不存在，则自动创建文件

f = open('LessionInfos.txt', 'w')

i=0for each inlessonInfos:

i+= 1f.writelines('第' + str(i) + '个课程：\n')

f.writelines('title:' + each['title'] + '\n')

f.writelines('desc:' + each['desc'] + '\n')

f.writelines('time:' + each['time'] + '\n')

f.writelines('level:' + each['level'] + '\n')

f.writelines('learnNumber:' + each['learnNumber'] + '\n\n')

f.close()if __name__ == '__main__':#定义课程信息数组

lessonInfos =[]#课程信息页面url

url = 'http://www.jikexueyuan.com/course/'

#实例化爬虫

spider =Spider()#取[1,21)及1到20页的课程信息

for i in range(1, 21):#构建分页URL

pageUrl = url + '?pageNum=' +str(i)print '正在处理页面：' +pageUrl

source=spider.getSource(pageUrl)

lessons=spider.getLessons(source)for lesson inlessons:

lessonInfo=spider.getLessonInfo(lesson)

lessonInfos.append(lessonInfo)#print 'title:'+lessonInfo.get('title')

#print 'desc:'+lessonInfo.get('desc')

#print 'time:'+lessonInfo.get('time')

#print 'level:'+lessonInfo.get('level')

#print 'learnNumber:'+lessonInfo.get('learnNumber')

print '已处理' + str(lessons.__len__()) + '个课程信息。'

print '极客学院课程信息爬取完毕，正在保存课程信息。。。'spider.saveLessionInfos(lessonInfos)print '极客学院课程信息保存完毕。'