Python爬取58足浴上网站信息

以58足浴（http://bj.58.com/zuyu/pn1/?PGTID=0d306b61-0000-186a-d0e6-09e79d939b21&ClickID=1）的该网页为例来实战下Xpath。想要爬取的信息包括：标题、类型、临近、转让费、租金、面积。

1. 使用基础库完成

先不使用框架，自己手写爬取该页的代码：

# -*- coding: utf-8 -*-
import codecs
import reimport pandas as pd
import sys
from lxml import etree
import requestsreload(sys)
sys.setdefaultencoding("utf-8")
# print res
# print type(res)
# with codecs.open("bj.58.html", mode="wb") as f:
#     f.write(res)
from lxml.etree import _Elementblank = ""  # 空字符串
colon_en = ":"  # 英文冒号
colon_zh = u"："  # 中文冒号
forward_slash = "/" # 正斜杠
br_label = "<br>"  # 换行标签pattern_space = re.compile("\s+")  # 空格
pattern_line = re.compile("<br\s*?/?>")  # 换行
pattern_label = re.compile("</?\w+[^>]*>")  # HTML标签def crawl_data(url):data = {"title": [],"kind": [],"approach": [],"trans_fee": [],"rent": [],"area": []}response = requests.get(url)res = response.contenttree = etree.HTML(res)frame = tree.xpath("//*[@id='infolist']/table/tr")# one = frame[0]# print one.xpath(".//text()")# print one.xpath("string()")for one in frame:# 标题提取 method 1raw_title = blank.join(one.xpath("./td[@class='t']/a/text()"))title = re.sub(pattern_space, blank, raw_title)# print("title: %s" % title)# method 2# title = one.xpath("string(./td[@class='t']/a)")data["title"].append(title)print("title: %s" % title)# 类型和临近位置提取raw_kind_and_approach = blank.join(one.xpath("./td[@class='t']/text()"))kind_and_approach = re.sub(pattern_space, blank, raw_kind_and_approach)k_and_a_list = kind_and_approach.split(forward_slash)kind = ""approach = ""for thing in k_and_a_list:if u"类型" in thing:kind = thing.split(colon_en)[1]elif u"临近" in thing:approach = thing.split(colon_en)[1]data["kind"].append(kind)data["approach"].append(approach)print("kind: %s, approach: %s" % (kind, approach))# 转让费和租金提取transfer_fee_and_rent = etree.tostring(one.xpath("./td[3]")[0], encoding="utf-8")# print("transfer_fee_and_rent: %s" % transfer_fee_and_rent)t_and_r_list = re.sub(pattern_space, blank, transfer_fee_and_rent).split(br_label)# 针对转让费为面议或租金为面议或都为面议的情况进行处理t_and_r_list = t_and_r_list if len(t_and_r_list) == 2 else t_and_r_list * 2transfer_fee = re.sub(pattern_label, blank, t_and_r_list[0]).split(colon_zh)[-1]rent = re.sub(pattern_label, blank, t_and_r_list[1]).split(colon_zh)[-1]data["trans_fee"].append(transfer_fee)data["rent"].append(rent)print("transfer_fee: %s, rent: %s" % (transfer_fee, rent))# 面积提取raw_area = etree.tostring(one.xpath("./td[position()=4]")[0], encoding="utf-8")area = re.sub(pattern_label, blank, raw_area)area = re.sub(pattern_space, blank, area)data["area"].append(area)print("area: %s" % area)print("-" * 50)# data.append(item)return datadef write_csv(data, file):df = pd.DataFrame(data)df.to_csv(file, index=False, encoding="gbk")if __name__ == "__main__":url = "http://bj.58.com/zuyu/pn1/?PGTID=0d306b61-0000-186a-d0e6-09e79d939b21&ClickID=1"data = crawl_data(url)out_file = "bj_58.csv"write_csv(data, out_file)# print("data: %s" % data)

运行后结果：

2. 使用Scrapy框架完成

命令行中输入 scrapy startproject tutorial来创建一个tutorial工程。

在items.py中添加一个新的Item：

class ZuYuItem(scrapy.Item):title = scrapy.Field()  # 标题kind = scrapy.Field()  # 类型approach = scrapy.Field()  # 临近transfer_fee = scrapy.Field()  # 转让费rent = scrapy.Field()  # 租金area = scrapy.Field()  # 面积

在spiders目录下创建一个名为 bj_58.py 的新的Python文件。内容如下：

# -*- coding: utf-8 -*-
import reimport scrapyfrom tutorial.items import ZuYuItemclass BJ58Spider(scrapy.Spider):"""scrapy crawl bj_58 -o res.csv"""name = "bj_58"start_urls = ["http://bj.58.com/zuyu/pn1/?PGTID=0d306b61-0000-186a-d0e6-09e79d939b21&ClickID=1"]def parse(self, response):blank = ""  # 空字符串colon_en = ":"  # 英文冒号colon_zh = u"："  # 中文冒号forward_slash = "/" # 正斜杠br_label = "<br>"  # 换行标签pattern_space = re.compile("\s+")  # 空格pattern_line = re.compile("<br\s*?/?>")  # 换行pattern_label = re.compile("</?\w+[^>]*>")  # HTML标签item = ZuYuItem()frame = response.xpath("//*[@id='infolist']/table/tr")# one = frame[0]# print one.xpath(".//text()").extract()  # 提取每个选择器所对应# print one.xpath("string()").extract_first()for one in frame:# 标题提取 method 1raw_title = blank.join(one.xpath("./td[@class='t']/a/text()").extract())title = re.sub(pattern_space, blank, raw_title)# method 2# title = one.xpath("string(./td[@class='t']/a)").extract_first()item["title"] = title# 类型和临近位置提取raw_kind_and_approach = blank.join(one.xpath("./td[@class='t']/text()").extract())kind_and_approach = re.sub(pattern_space, blank, raw_kind_and_approach)k_and_a_list = kind_and_approach.split(forward_slash)kind = ""approach = ""for thing in k_and_a_list:if u"类型" in thing:kind = thing.split(colon_en)[1]elif u"临近" in thing:approach = thing.split(colon_en)[1]item["kind"] = kinditem["approach"] = approach# 转让费和租金提取transfer_fee_and_rent = one.xpath("./td[position()=3]").extract_first()t_and_r_list = re.sub(pattern_space, blank, transfer_fee_and_rent).split(br_label)self.log("title: %s" % title)self.log("t_and_r_list: %s" % t_and_r_list)t_and_r_list = t_and_r_list if len(t_and_r_list) == 2 else t_and_r_list * 2self.log("t_and_r_list: %s" % t_and_r_list)transfer_fee = re.sub(pattern_label, blank, t_and_r_list[0]).split(colon_zh)[-1]rent = re.sub(pattern_label, blank, t_and_r_list[1]).split(colon_zh)[-1]item["transfer_fee"] = transfer_feeitem["rent"] = rent# 面积提取raw_area = one.xpath("./td[position()=4]").extract_first()area = re.sub(pattern_label, blank, raw_area)item["area"] = areayield item

在命令行中输入 scrapy crawl bj_58 -o res.csv 将结果存入res.csv文件中

Python爬取58足浴上网站信息相关推荐

python爬取58同城的兼职信息
标题python爬取58同城的兼职信息刚刚开始学习爬虫,一些自己研究的小东西,爬取58同城上面的兼职信息放入Excel表格里,具体代码解释在代码里给出注释,下面给出完整代码: #首先是导包 imp ...
python实战｜python爬取58同城租房数据并以Excel文件格式保存到本地
python实战|python爬取58同城租房数据并以Excel文件格式保存到本地一.分析目标网站url 目标网站:https://cq.58.com/minsuduanzu/ 让我们看看网站长啥样 ...
Python爬取58同城广州房源+可视化分析
感谢关注天善智能,走好数据之路↑↑↑ 欢迎关注天善智能,我们是专注于商业智能BI,人工智能AI,大数据分析与挖掘领域的垂直社区,学习,问答.求职一站式搞定! 对商业智能BI.大数据分析挖掘.机器学习, ...
利用python爬取58同城简历数据
利用python爬取58同城简历数据最近接到一个工作,需要获取58同城上面的简历信息(http://gz.58.com/qzyewu/).最开始想到是用python里面的scrapy框架制作爬虫.但 ...
利用python爬取58同城简历数据_利用python爬取58同城简历数据-Go语言中文社区
利用python爬取58同城简历数据最近接到一个工作,需要获取58同城上面的简历信息(http://gz.58.com/qzyewu/).最开始想到是用python里面的scrapy框架制作爬虫.但 ...
深夜，我用python爬取了整个斗图网站，不服来斗
深夜,我用python爬取了整个斗图网站,不服来斗 QQ.微信斗图总是斗不过,索性直接来爬斗图网,我有整个网站的图,不服来斗. 废话不多说,选取的网站为斗图啦,我们先简单来看一下网站的结构网页信息 ...
python爬房源信息_用python爬取链家网的二手房信息
题外话:这几天用python做题,算是有头有尾地完成了.这两天会抽空把我的思路和方法,还有代码贴出来,供python的初学者参考.我python的实战经历不多,所以代码也是简单易懂的那种.当然过程中还 ...
python爬取boss直聘招聘信息_年底啦，利用Python爬取Boss直聘的招聘信息，为明年跳槽做准备...
原标题:年底啦,利用Python爬取Boss直聘的招聘信息,为明年跳槽做准备前言为什么都说程序员的薪资高,但是刚开始入职的你,薪资并不是自己想象中的那样,首先是你的工作经验不足,其次就是需要不断的 ...
python 爬取链家数据_用python爬取链家网的二手房信息
题外话:这几天用python做题,算是有头有尾地完成了.这两天会抽空把我的思路和方法,还有代码贴出来,供python的初学者参考.我python的实战经历不多,所以代码也是简单易懂的那种.当然过程中还 ...
Python爬取豆瓣电影top250的电影信息
Python爬取豆瓣电影top250的电影信息前言一.简介二.实例源码展示小结前言相信很多小伙伴在学习网络爬虫时,老师们会举一些实例案例讲解爬虫知识,本文介绍的就是经典爬虫实际案例–爬取豆 ...

Python爬取58足浴上网站信息

Python爬取58足浴上网站信息相关推荐

最新文章

热门文章