#  本文以浦东新区为例--其他区自行举一反三

import requests
import pandas as pd
import pprint
import csv
import pandas as pd
from bs4 import BeautifulSoup
url='https://sh.lianjia.com/chengjiao/{areo}/pg{page}/'
data=[]
#定义区列表
arealist=['beicai','biyun','caolu','chuansha','datuanzhen','geqing','gaohang','gaodong','huamu','hangtou','huinan','jinqiao','jinyang','kangqiao','lujiazui','laogangzhen','lingangxincheng','lianyang','nichengzhen','nanmatou','sanlin','shibo','shuyuanzhen','tangqiao','tangzhen','waigaoqiao','wanxiangzhen','weifang','xuanqiao','xinchang','yuqiao1','yangdong','yuanshen','yangjing','zhangjiang','zhuqiao','zhoupu']
for j in range (1,101):for a in arealist:houseurl=url.format(areo=a,page=j)res=requests.get(houseurl)res.encoding='utf-8'soup=BeautifulSoup(res.text,'html.parser')content=soup.select('.info')#     print(content)#     print (type(content))for element in content:datum={}try:title=element.select('.title')[0].textexcept:title=''try:houseinfo=element.select('.houseInfo')[0].textexcept:houseinfo=''try:dealDate=element.select('.dealDate')[0].textexcept:houseinfo=''try:positionInfo=element.select('.positionInfo')[0].textexcept:positionInfo=''try:unitPrice=element.select('.unitPrice')[0].textexcept:unitPrice=''try:showprice=element.select('.dealCycleTxt')[0].textexcept:showprice=''try:totalPrice=element.select('.totalPrice')[0].textexcept:totalPrice=''try:metroline=element.select('.dealHouseTxt')[0].textexcept:metroline=''datum['title']=titledatum['metroline']=metrolinedatum['houseinfo']=houseinfodatum['dealDate']=dealDatedatum['positionInfo']=positionInfodatum['unitPrice']=unitPricedatum['showprice']=showpricedatum['totalPrice']=totalPricedatum['page']=jdatum['area']='pudong'   #属于哪个区放进去datum['mingxi']=a        #区下面的具体街道data.append(datum)
#             pprint.pprint(datum)

# 保存成csv文件
df=pd.DataFrame(data)
# print(data)
df.head()
df.to_csv('C/Sandra/pudong.csv')