python 爬取中国房价行情网网站

代码下载 https://github.com/tanjunchen/SpiderProject/tree/master/HouseWorldNewHouse

HousePriceSpider.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import json
import time
from multiprocessing import Pool, Manager, cpu_count
from urllib.parse import urljoin
import numpy as np
import pandas as pd
import hashlib
from lxml import etree
from ChinaHousePrice.session import SessionWrapper
from datetime import datetime
from ChinaHousePrice.common import AreaCodeDecoder
import pymysql
from sqlalchemy import create_engine
from ChinaHousePrice import config
from sqlalchemy.sql import textclass Spider(object):def __init__(self, prov, city, url):self.prov = provself.city = cityself.url = urlself.__session = SessionWrapper(timeout=20)self.__href_d = self._district()self.city_data = self._city_data()def _district(self):href_d = {}res = self.__session.get(self.url)if res is None:return href_dhtml = etree.HTML(res.text)href_n = html.xpath("//span[@class='city-n']/a/@href")district_name_n = html.xpath("//span[@class='city-n']/a/span/text()")href_d.update(dict(zip(district_name_n, href_n)))href_w = html.xpath("//span[@class='city-w']/a/@href")district_name_w = html.xpath("//span[@class='city-w']/a/span/text()")href_d.update(dict(zip(district_name_w, href_w)))return href_ddef _city_data(self):return self._district_all_data(np.nan, self.url + "?")def _district_all_data(self, district, url):params = ["", "?&type=newha", "?&type=lease", "?&type=lease&proptype=22", "?&proptype=22", "?&proptype=21","?&type=lease&proptype=22"]result = [self.prov, self.city, district]for u in params:url_ = url + uresult += self._parse_by_xpath(url_)print("Crawl-->" + url_, self._parse_by_xpath(url_))# prov,city,district,住宅+二手房+价格,住宅+二手房+环比,住宅+新楼盘+价格,住宅+新楼盘+环比,住宅+出租+价格,住宅+出租+环比,# 商铺+二手房+价格,商铺+二手房+环比,商铺+出租+价格,商铺+出租+环比,# 办公+二手房+价格,办公+二手房+环比,办公+出租+价格,办公+出租+环比return resultdef _parse_by_xpath(self, url):try:response = self.__session.get(url)if response and response.ok:html = etree.HTML(response.text)auto = html.xpath("//span[@id='viewkey_wp']")if len(auto) > 0:print("\033[31m弹出验证码了......赶紧去解决它.....\033[0m")ul = html.xpath("//*[@id='content']/div[4]/div[1]/div[2]/div//div/ul")if len(ul) > 1:data = (ul[1].xpath("./li/span/text()"))# print(data)if len(data) != 2:raise ValueErrorif len(data) == 2:data[0] = data[0].replace(",", "")data[1] = data[1].replace(",", "")return dataelif len(ul) == 1:data = (ul[0].xpath("./li/span/text()"))if len(data) == 1:if data[0] == '--':data[0] = np.nandata.append(np.nan)if len(data) != 2:raise ValueErrorreturn dataraise ValueErrorexcept ValueError:return [np.nan, np.nan]except Exception:SessionWrapper.to_exception(url)return [np.nan, np.nan]def run(self):result = [self.city_data]for n, h in self.__href_d.items():url = urljoin(self.url, h)result.append(self._district_all_data(n, url))df = pd.DataFrame(result)df.columns = ["省", "市", "区", "住宅:二手房:价格", "住宅:二手房:环比", '住宅:新楼盘:价格', '住宅:新楼盘:环比', "住宅:出租:价格", "住宅:出租:环比","商铺:二手房:价格", "商铺:二手房:环比", "商铺:出租:价格", "商铺:出租:环比","办公:二手房:价格", "办公:二手房:环比", "办公:出租:价格", "办公:出租:环比"]return dfdef thread(_prov, _city, _url, _list):s = Spider(_prov, _city, _url)_list.append(s.run())def job(target=None):start = time.time()with open("url/format_cre_adjust.json", encoding="utf-8") as file:d = json.load(file)header = Truefor i, prov in enumerate(d):last = time.time()if target is not None and prov not in target:continuepool_size = min(cpu_count() * 4, len(d[prov]))result_list = Manager().list()pool = Pool(pool_size)for city in d[prov]:pool.apply_async(thread,args=(prov, city, d[prov][city], result_list),error_callback=lambda e: print(e))pool.close()pool.join()df = pd.concat(result_list)df.to_csv("des_data/all_cre_result_" + datetime.now().strftime('%Y-%m-%d') + ".csv",mode="w" if header else "a+", header=header, index=False,encoding="gbk")header = Falseprint(str((i + 1) / len(d) * 100)[0:5], "%", prov, "耗时", time.time() - last, "秒")time.sleep(5)print("任务总耗时", time.time() - start, "秒")'''数据分析 导入数据到数据库'''def item_name(address, name):address = address.replace(" ", "").replace(",", ":")return address + ":" + name if address[-1] != ":" else address[:-1] + ":" + namedef get_conn():conn = pymysql.connect(host=config.host, port=config.port, user=config.user, passwd=config.passwd,db=config.db, charset='utf8')return conndef analysis():now_x = datetime.now().strftime('%Y-%m-%d')df = pd.read_csv("des_data/all_cre_result_" + now_x + ".csv", encoding='gbk')# df = pd.read_csv("des_data/all_cre_result_2018-11-30.csv", encoding='gbk')df = df.rename(columns={"住宅:二手房:价格": "二手房价格:住宅", "住宅:新楼盘:价格": "新楼盘价格:住宅", "住宅:出租:价格": "租金:住宅","商铺:二手房:价格": "二手房价格:商铺", "商铺:出租:价格": "租金:商铺", "办公:二手房:价格": "二手房价格:办公","办公:出租:价格": "租金:办公"})index_names = ['省', '市', '区', '二手房价格:住宅', '新楼盘价格:住宅', '租金:住宅', '二手房价格:商铺', '租金:商铺', '二手房价格:办公', '租金:办公']# df_all所有的数据df_all = df.loc[:, index_names]df_all = pd.melt(df_all, id_vars=['省', '市', '区'], var_name='index_name', value_name='value')df_all['区'] = df_all['区'].fillna(" ")df_all['地址'] = df_all.loc[:, '省'] + "," + df_all.loc[:, '市'] + "," + df_all.loc[:, '区']a = AreaCodeDecoder()df_all['areacode'] = df_all['地址'].apply(lambda address: a.format_addr_code(address))df_all['item_name'] = df_all.apply(lambda df_all: item_name(df_all['地址'], df_all['index_name']), axis=1)df_all["item_id"] = df_all['item_name'].map(lambda address: "c" + str(hashlib.md5(address.encode()).hexdigest()))df_all["freq"] = "W"df_all["data_source"] = "HousePrice"rename_dict = {"index_name": "item_short_name","value": "fvalue",}df_all = df_all.rename(columns=rename_dict)df_all['unit'] = df_all["item_short_name"].apply(lambda x: '元/平方米' if x in ['二手房价格:住宅', '新楼盘价格:住宅', '二手房价格:商铺', '二手房价格:办公'] else '元/月/平方米')# 房价网的itemdf_all_item = df_all.loc[:, ["item_id", "item_name", "freq", "unit", "areacode", "item_short_name","data_source"]].drop_duplicates()# 房价网的数据df_all_data = df_all.loc[:, ["item_id", "fvalue"]].dropna().drop_duplicates()df_all_item = df_all_item.loc[(df_all_item["item_id"].isin(df_all_data["item_id"]))]df_all_data['fvalue'] = df_all_data['fvalue'].apply(lambda x: str(x).replace(',', '')).astype('float64')df_all_data['fdate'] = now_xengine = create_engine(config.conn_tools)exists_item = pd.read_sql("select item_id from " + config.table_item_name + " where data_source = 'HousePrice'",engine)# 新的itemnew_item = df_all_item.loc[np.logical_not(df_all_item["item_id"].isin(exists_item["item_id"]))]new_item["start_date"] = now_xdef tx(conn, new_item_x, df_all_data_x, df_all_item_x, now_xx):# 更新Itemnew_item_x.to_sql(config.table_item_name, conn, index=False, if_exists="append")# 更新数据df_all_data_x.to_sql(config.table_item_data, conn, index=False, if_exists="append")for item_id in df_all_item_x["item_id"].values:conn.execute(text("update " + config.table_item_name + " set end_date = :date where item_id = :item_id"),date=now_xx, item_id=item_id)try:engine.transaction(tx, new_item, df_all_data, df_all_item, now_x)print("插入数据成功")except Exception as e:print(e)print("插入数据失败")if __name__ == '__main__':job()  # 开始房价网的数据的抓取analysis()  # 数据分析与数据插入到数据库中

完整代码请查看 https://github.com/tanjunchen/SpiderProject/tree/master/HouseWorldNewHouse

python 爬取中国房价行情网网站相关推荐

python中国最好大学排名_国内大学排名如何？用Python爬取中国大学排名
国内大学排名如何?用Python爬取中国大学排名准备阶段需要的库robots协议上代码代码框架*获取url信息*解析信息*输出数据*主函数结果准备阶段新手入门,不喜勿喷,这篇文章的内容其实也是在中 ...
python爬取中国天气网天气图标
python爬取中国天气网天气图标准备工作天气预报图例网址:http://www.weather.com.cn/static/html/legend.shtml 安装requests:pip in ...
python爬取中国大学（高校）基本信息
Python爬取中国大学(高校)基本信息 python爬取中国大学(高校)基本信息简单的一个小爬虫,获取中国高校基本信息一.输出到excel表格结果二.代码 // An highlighted ...
深夜，我用python爬取了整个斗图网站，不服来斗
深夜,我用python爬取了整个斗图网站,不服来斗 QQ.微信斗图总是斗不过,索性直接来爬斗图网,我有整个网站的图,不服来斗. 废话不多说,选取的网站为斗图啦,我们先简单来看一下网站的结构网页信息 ...
Python爬取中国知网文献、参考文献、引证文献
转载自博客园文章作为学习资料,代码及相关介绍非常详细.原文链接见Python爬取中国知网文献.参考文献.引证文献
用Python爬取中国各省GDP数据
介绍在数据分析和经济研究中,了解中国各省份的GDP数据是非常重要的.然而,手动收集这些数据可能是一项繁琐且费时的任务.幸运的是,Python提供了一些强大的工具和库,使我们能够自动化地从互联网上爬取 ...
python爬取中国天气网中国全部城市的天气链接
近期自己尝试用python写一个qqbot实现天气查询功能,现已基本实现已经基本完成爬取中国天气网的所有城市的天气情况链接下附中国天气网所有城市的名称及对应链接 {'朝阳': 'http://ww ...
Python爬取中国大学排名，并且保存到excel中
前言以下文章来源于数据分析和Python ,作者冈坂日川今天发的是python爬虫爬取中国大学排名,并且保存到excel中,当然这个代码很简单,我用了半小时就写完了,我的整体框架非常清晰,可以直接 ...
使用Python爬取中国Mooc网讨论区内所有评论
最近受朋友委托,写了一个爬取中国Mooc往讨论区所有评论的程序 ,他需要这些数据写论文. 如下: 他需要爬取这些板块里所有的评论. 俗话说的好六月七月爬虫热因为这段时间许多的大学生都毕业了特此写 ...

python 爬取中国房价行情网网站

python 爬取中国房价行情网网站相关推荐

最新文章

热门文章