09批量提取京东商品名称和价格

import requests
from lxml import etree
import time
import csv
from urllib.request import urlopen, Request
from lxml import etreedef crow_first(n):# 定义函数抓取每页前30条商品信息# 构造每一页的url变化url = 'https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&cid2=653&cid3=655&page='head = {'authority': 'search.jd.com','method': 'GET','path': '/s_new.php?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&page=4&s=84&scrolling=y&log_id=1529828108.22071&tpl=3_M&show_items=7651927,7367120,7056868,7419252,6001239,5934182,4554969,3893501,7421462,6577495,26480543553,7345757,4483120,6176077,6932795,7336429,5963066,5283387,25722468892,7425622,4768461','scheme': 'https','referer': 'https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&page=3&s=58&click=0','user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36','x-requested-with': 'XMLHttpRequest','Cookie': 'qrsc=3; pinId=RAGa4xMoVrs; xtest=1210.cf6b6759; ipLocation=%u5E7F%u4E1C; _jrda=5; TrackID=1aUdbc9HHS2MdEzabuYEyED1iDJaLWwBAfGBfyIHJZCLWKfWaB_KHKIMX9Vj9_2wUakxuSLAO9AFtB2U0SsAD-mXIh5rIfuDiSHSNhZcsJvg; shshshfpa=17943c91-d534-104f-a035-6e1719740bb6-1525571955; shshshfpb=2f200f7c5265e4af999b95b20d90e6618559f7251020a80ea1aee61500; cn=0; 3AB9D23F7A4B3C9B=QFOFIDQSIC7TZDQ7U4RPNYNFQN7S26SFCQQGTC3YU5UZQJZUBNPEXMX7O3R7SIRBTTJ72AXC4S3IJ46ESBLTNHD37U; ipLoc-djd=19-1607-3638-3638.608841570; __jdu=930036140; user-key=31a7628c-a9b2-44b0-8147-f10a9e597d6f; areaId=19; __jdv=122270672|direct|-|none|-|1529893590075; PCSYCityID=25; mt_xid=V2_52007VwsQU1xaVVoaSClUA2YLEAdbWk5YSk9MQAA0BBZOVQ0ADwNLGlUAZwQXVQpaAlkvShhcDHsCFU5eXENaGkIZWg5nAyJQbVhiWR9BGlUNZwoWYl1dVF0%3D; __jdc=122270672; shshshfp=72ec41b59960ea9a26956307465948f6; rkv=V0700; __jda=122270672.930036140.-.1529979524.1529984840.85; __jdb=122270672.1.930036140|85.1529984840; shshshsID=f797fbad20f4e576e9c30d1c381ecbb1_1_1529984840145'}ret = Request(url, headers=head)res = urlopen(ret)html = res.read().decode('utf-8')html = etree.HTML(html)# 定位到每一个商品标签lidatas = html.xpath('//li[contains(@class,"gl-item")]')with open('JD_Phone.csv', 'a', newline='', encoding='utf-8')as f:write = csv.writer(f)for data in datas:p_price = data.xpath('div/div[@class="p-price"]/strong/i/text()')p_comment = data.xpath('div/div[5]/strong/a/text()')p_name = data.xpath('div/div[@class="p-name p-name-type-2"]/a/em')# 这个if判断用来处理那些价格可以动态切换的商品，比如上文提到的小米MIX2，他们的价格位置在属性中放了一个最低价if len(p_price) == 0:p_price = data.xpath('div/div[@class="p-price"]/strong/@data-price')# xpath('string(.)')用来解析混夹在几个标签中的文本# print(p_name[0].xpath('string(.)'))# print(p_price[0])# print(p_comment[0])write.writerow([p_name[0].xpath('string(.)'),p_price[0]])f.close()def crow_last(n):# 获取当前的Unix时间戳，并且保留小数点后5位a = time.time()b = '%.5f' % aurl = 'https://search.jd.com/s_new.php?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&page=' + str(2 * n) + '&s=' + str(48 * n - 20) + '&scrolling=y&log_id=' + str(b)head = {'authority': 'search.jd.com','method': 'GET','path': '/s_new.php?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA','scheme': 'https','referer': 'https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&enc=utf-8&qrst=1&rt=1&stop=1&vt=2&wq=%E6%89%8B%E6%9C%BA&cid2=653&cid3=655&page=3&s=58&click=0','user-agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/66.0.3359.139 Safari/537.36','x-requested-with': 'XMLHttpRequest','Cookie': 'qrsc=3; pinId=RAGa4xMoVrs; xtest=1210.cf6b6759; ipLocation=%u5E7F%u4E1C; _jrda=5; TrackID=1aUdbc9HHS2MdEzabuYEyED1iDJaLWwBAfGBfyIHJZCLWKfWaB_KHKIMX9Vj9_2wUakxuSLAO9AFtB2U0SsAD-mXIh5rIfuDiSHSNhZcsJvg; shshshfpa=17943c91-d534-104f-a035-6e1719740bb6-1525571955; shshshfpb=2f200f7c5265e4af999b95b20d90e6618559f7251020a80ea1aee61500; cn=0; 3AB9D23F7A4B3C9B=QFOFIDQSIC7TZDQ7U4RPNYNFQN7S26SFCQQGTC3YU5UZQJZUBNPEXMX7O3R7SIRBTTJ72AXC4S3IJ46ESBLTNHD37U; ipLoc-djd=19-1607-3638-3638.608841570; __jdu=930036140; user-key=31a7628c-a9b2-44b0-8147-f10a9e597d6f; areaId=19; __jdv=122270672|direct|-|none|-|1529893590075; PCSYCityID=25; mt_xid=V2_52007VwsQU1xaVVoaSClUA2YLEAdbWk5YSk9MQAA0BBZOVQ0ADwNLGlUAZwQXVQpaAlkvShhcDHsCFU5eXENaGkIZWg5nAyJQbVhiWR9BGlUNZwoWYl1dVF0%3D; __jdc=122270672; shshshfp=72ec41b59960ea9a26956307465948f6; rkv=V0700; __jda=122270672.930036140.-.1529979524.1529984840.85; __jdb=122270672.1.930036140|85.1529984840; shshshsID=f797fbad20f4e576e9c30d1c381ecbb1_1_1529984840145'}r = requests.get(url, headers=head)r.encoding = 'utf-8'html1 = etree.HTML(r.text)datas = html1.xpath('//li[contains(@class,"gl-item")]')with open('JD_Phone.csv', 'a', newline='', encoding='utf-8')as f:write = csv.writer(f)for data in datas:p_price = data.xpath('div/div[@class="p-price"]/strong/i/text()')p_comment = data.xpath('div/div[5]/strong/a/text()')p_name = data.xpath('div/div[@class="p-name p-name-type-2"]/a/em')if len(p_price) == 0:p_price = data.xpath('div/div[@class="p-price"]/strong/@data-price')write.writerow(p_price[0])write.writerow(p_name[0].xpath('string(.)'))f.close()if __name__ == '__main__':for i in range(1, 3):crow_first(i)

09批量提取京东商品名称和价格相关推荐

【R的网络提取】什么值得买批量提取商品信息和价格等数据
由于电商行业经常需要对价格进行监控,而什么值得买本身就是一个比价网站,那么这个网站对很多电商具有一定的参考性:本文用R进行什么值得买的相关信息提取,针对这个信息,可以对商品走势进行详细分析:当然,另一 ...
用selenium爬取京东平台商品列表,爬取商品名称、价格、店铺信息
#用selenium爬取京东平台商品列表,爬取商品名称.价格.店铺信息from selenium import webdriver from selenium.webdriver.common.by ...
Python批量提取Excel文件中的图片
推荐图书: <Python程序设计(第3版)>,(ISBN:978-7-302-55083-9),董付国,清华大学出版社,2020年6月第1次印刷,2021年7月第9次印刷,山东省一流本科 ...
【Python】爬虫入门级实战讲解：爬取商城的商品名称及价格
需求: 1.对于某某商城,爬取其商品名称和商品价格 2.要求:用户动态输入爬取特定商品: 3.爬取结果导入excel表格,并写入数据库中: 思路: 1.查看搜索商品的接口信息,包括接口名称,接口方法( ...
寻找与疾病相关的SNP位点——R语言从SNPedia批量提取搜索数据
SNP是单核苷酸多态性,人的基因是相似的,有些位点上存在差异,这种某个位点的核苷酸差异就做单核苷酸多态性,它影响着生物的性状,影响着对某些疾病的易感性.SNPedia是一个SNP调査百科,它引用 ...
word数据提取技巧：如何批量提取身份证出生日期
在实际工作中,我们时常在整理员工资料或客户信息时需要从身份证号中提取对应人员的出生日期等看似重复繁琐的工作,是不是瞬间觉得工作量好大. 机智的同学,第一时间会想到使用Excel公式或函数来提取整理.但 ...
批量提取视频帧率及分辨率
最近遇到一个需求,需要做一些frame rate conversion相关的工作,首先分析需求,有哪些FRC类型?这里需要批量提取一批文件的帧率及分辨率. 1.帧率查看工具遇到视频,很难不想到ffm ...
个人永久性免费-Excel催化剂功能第84波-批量提取OUTLOOK邮件附件
批量操作的事情常常能让人感到十分畅快,区别于一次次的手工的操作,它真正实现了"人工智能"想要的效果,人指挥机器做事情,机器就可以按着人意去操作. 此篇给大家再次送了批量操作一绝活, ...
个人永久性免费-Excel催化剂功能第88波-批量提取pdf文件信息（图片、表格、文本等）...
日常办公场合中,除了常规的Excel.Word.PPT等文档外,还有一个不可忽略的文件格式是pdf格式,而对于想从pdf文件中获取信息时,常规方法将变得非常痛苦和麻烦.此篇给大家送一pdf文件提取信息 ...

09批量提取京东商品名称和价格

09批量提取京东商品名称和价格相关推荐

最新文章

热门文章