使用Python进行上海市租房信息爬取，通过requests + Beautifulsoup对网页内容进行抓取和数据提取。

import requests

from bs4 import BeautifulSoup

headers = {'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/53.0.2785.104 Safari/537.36 Core/1.53.2669.400 QQBrowser/9.6.10990.400',

'Accept':'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',

'Accept-Encoding':'gzip, deflate, sdch',

'Accept-Language':'zh-CN,zh;q=0.8',

'Connection':'keep-alive',

'Referer':'https://www.baidu.com/link?url=znRkKNPxTXXmN7m4lWy28vWXLWyx_hTEW4bLC5Pi8YR_s5wAdQk4FExi8Fz6HxHc&wd=&eqid=f2ad9a7e00007f1200000004594a3116'

}

urls = [] # 定义空列表，用于创建所有的爬虫链接

#基于for循环，构造完整的爬虫链接

for i in ['zufang']:

url = 'http://sh.lianjia.com/%s/' %i

for j in list(range(1,2075)):

urls.append('http://sh.lianjia.com/%s/d%si1s5' %(i,j)) #拼接所有需要爬虫的链接

#创建csv文件，用于后面的保存数据

file = open('D:/jupyter/lianjia.csv','w')

def readpage(url):

try:

res = requests.get(url)

except ConnectionError as e:

return None

return res

def viewsubway(res2):

try:

view = res2.find_all('span',{'class':'fang-subway-ex'})[0].find_all('span')[0].text

except IndexError as e:

return None

return view

for url in urls: #基于for循环，抓取出所有满足条件的标签和属性列表，存放在find_all中

res = readpage(url)

res = res.text.encode(res.encoding).decode('utf-8')

soup = BeautifulSoup(res,'html.parser')

find_all = soup.find_all(name = 'div',attrs = {'class':'info-panel'})

for i in list(range(len(find_all))): #基于for循环，抓取出所需的各个字段信息

title = find_all[i].find('a')['title'] #每套租房的标语

res2 = find_all[i]

name = res2.find_all('div',{'class':'where'})[0].find_all('span')[0].text #租房所在的小区名称

room_type = res2.find_all('div',{'class':'where'})[0].find_all('span')[1].text #租房的户型

size = res2.find_all('div',{'class':'where'})[0].find_all('span')[2].text[:-3]#租房的面积

#采用列表解析式，删除字符串的首位空格

info = [i.strip() for i in res2.find_all('div',{'class':'con'})[0].text.split('\n')]

region = info[1] #租房所在的区

area = info[2] #租房所在的街道地域

louceng = info[4][:2] #租房所在的楼层

chaoxiang = info[7][3:] #租房的朝向

#每套租房的租金

price = res2.find_all('div',{'class':'price'})[0].find_all('span')[0].text

#地铁站及地铁

view = viewsubway(res2)

if view != None:

subway = view[2:5] #租房附近的地铁线

distance = view[-4:-1] #租房到地铁站的距离

else:

subway = '\0'

distance = '\0'

#将上面的各字段信息值写入并保存到csv文件中

file.write(','.join((name.encode('gbk','ignore'),room_type.encode('gbk','ignore'),size.encode('gbk','ignore'),region.encode('gbk','ignore'),area.encode('gbk','ignore'),louceng.encode('gbk','ignore'),chaoxiang.encode('gbk','ignore'),price.encode('gbk','ignore'),subway.encode('gbk','ignore'),distance.encode('gbk','ignore'),title.encode('gbk','ignore'))) + '\n')

关闭文件(否则数据不会写入到csv文件中)

file.close()```