Python爬取晨星网的基金数据

# -*- coding: utf-8 -*-import sys
from lxml import etree
import time
import random
import requests
import json
import csv
from selenium import webdriver   # 导入webdriver模块
from bs4 import BeautifulSoup
from time import sleepclass ChengxingspiderItem:def __init__(self):self.fund_code = ""self.fund_name = ""self.fund_price = ""self.category = ""self.inception = ""self.subscribe = ""self.redeem = ""self.sbdesc = ""self.sharprate = ""self.date = ""self._i = 0def __iter__(self):return selfdef __next__(self):if self._i == 0:self._i += 1return self.fund_codeelif self._i == 1:self._i += 1return self.fund_nameelif self._i == 2:self._i += 1return self.fund_priceelif self._i == 3:self._i += 1return self.categoryelif self._i == 4:self._i += 1return self.sharprateelif self._i == 5:self._i += 1return self.dateelse:raise StopIteration()class FundSpider():name = 'fund'allowed_domains = ['cn.morningstar.com/quicktake/']# fund_list = ['0P0000Z821', 'F0000003VJ', '0P00016WFU', 'F000000416', 'F0000004AI',#             '0P00015HFT', 'F0000003ZX', '0P000147K8', '0P0000P5UD', '0P0001606X']fund_list = ['0P0000RU7I', '0P0000S0NU', '0P0000XBF0', '0P0000YXTA', '0P0000Z5JW', '0P0000ZEAH', '0P00015WGK', '0P0001606X', '0P000160TK', '0P00016A08', '0P00016DKC','0P00016FT6', '0P000178CP', '0P00018J2K', '0P00018KU4', '0P0001ABM2', '0P0001D6IU', '0P0001F1K9', '0P0001FDKQ', 'F0000003VJ', 'F0000004AI', 'F0000004JE']start_urls = [f'http://cn.morningstar.com/quicktake/{page}' for page in fund_list]# json_url = 'http://cn.morningstar.com/handler/quicktake.ashx'url = 'http://cn.morningstar.com/handler/quicktake.ashx'webUrl = 'http://cn.morningstar.com/quicktake/'def parse(self, response):# 把网页变成xpath结构fund_xpath = etree.HTML(response)item = ChengxingspiderItem()fund_name = fund_xpath.xpath('//*[@id="qt_fund"]/span[1]/text()')fund_price = fund_xpath.xpath('//*[@id="qt_base"]/ul[1]/li[2]/span/text()')date = fund_xpath.xpath('//*[@id="qt_base"]/ul[1]/li[3]/text()')category = fund_xpath.xpath('//*[@id="qt_base"]/ul[3]/li[7]/span/text()')sharprate = fund_xpath.xpath('//*[@id="qt_risk"]/li[30]/text()')item.fund_code = fund_name[0][:6]item.fund_name = fund_name[0][7:]item.fund_price = fund_price[0]if len(category) != 0:item.category = category[0]if len(sharprate) != 0:item.sharprate = sharprate[0]item.date = date[0][5:]print(item.fund_code)print(item.fund_name)print(item.fund_price)print(item.category)print(item.sharprate)# print(len(date[0]))# print(type(item['date']))# print(type(fund_price))# print(dir(fund_price))# print(len(str(date)))# print(str(date)[-12:-1])return list(item)if __name__ == '__main__':# headers = {'User-Agent':#           'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.105 Safari/537.36',#           'Host': 'cn.morningstar.com'#          }fs = FundSpider()# 指定chrom的驱动# 执行到这里的时候Selenium会到指定的路径将chrome driver程序运行起来# driver = webdriver.Chrome('./chromedriver/chromedriver.exe')# driver = webdriver.Firefox()#这里是火狐的浏览器运行方法options = webdriver.ChromeOptions()options.add_argument("--no-sandbox")driver = webdriver.Chrome('./chromedriver/chromedriver.exe', chrome_options=options)cookies = ''headers = ['fund_code', 'fund_name','fund_price', 'category', 'sharprate', 'date']rows = []for urlitem in fs.start_urls:# soup = BeautifulSoup(html_doc, 'html.parser')if(cookies == ''):# 可以通过 id 获取 form 表单driver.get(urlitem)username = driver.find_element_by_id('emailTxt')password = driver.find_element_by_id('pwdValue')username.send_keys('***********@qq.com')password.send_keys('w*************')submit = driver.find_element_by_id('loginGo')submit.click()sleep(1)# 获取网站cookiecookies = driver.get_cookies()else:# driver.add_cookie(cookies)driver.get(urlitem)singlerow = fs.parse(driver.page_source)rows.append(singlerow)# reobj = requests.get(urlitem, headers=headers)  # 对HTTP响应的数据# fs.parse(reobj)with open('test.csv', 'w', newline='')as f:f_csv = csv.writer(f)f_csv.writerow(headers)f_csv.writerows(rows)

如图：

CSV示例

import csvheaders = ['class', 'name', 'sex', 'height', 'year']rows = [[1, 'xiaoming', 'male', 168, 23],[1, 'xiaohong', 'female', 162, 22],[2, 'xiaozhang', 'female', 163, 21],[2, 'xiaoli', 'male', 158, 21]
]with open('test.csv', 'w',newline='')as f:f_csv = csv.writer(f)f_csv.writerow(headers)f_csv.writerows(rows)

如图：

下载地址

Python爬取晨星网的基金数据相关推荐

python爬取火车票网的时刻表数据
python爬取火车票网的时刻表数据导包 import re,requests,datetime,time,json from prettytable import PrettyTable from ...
Python爬取天气网历史天气数据
我的第一篇博客,哈哈哈,记录一下我的Python进阶之路! 今天写了一个简单的爬虫. 使用Python的requests 和BeautifulSoup模块,Python 2.7.12可在命令行中直接使 ...
使用python爬取东方财富网机构调研数据
最近有一个需求,需要爬取东方财富网的机构调研数据.数据所在的网页地址为: 机构调研网页如下所示: 可见数据共有8464页,此处不能直接使用scrapy爬虫进行爬取,因为点击下一页时,浏览器只是发起了 ...
python培训机构调研最多的股票_使用python爬取东方财富网机构调研数据
标签: 最近有一个需求,需要爬取东方财富网的机构调研数据.数据所在的网页地址为: 机构调研网页如下所示: 可见数据共有8464页,此处不能直接使用scrapy爬虫进行爬取,因为点击下一页时,浏览器只 ...
python二手房价格预测_Python爬取赶集网北京二手房数据R对爬取的二手房房价做线性回归分析...
前言:本文主要分为两部分:Python爬取赶集网北京二手房数据&R对爬取的二手房房价做线性回归分析.文章思路清晰,代码详细,特别适合刚刚接触Python&R的同学学习参考. Part1 ...
python爬取东方财富网资金流向数据（在本地生成csv文件）
今天我们来试着用python爬取东方财富网资金流向的表格数据. 第一步:程序及应用的准备首先我们需要安装selenium库,使用命令pip install selenium;然后我们需要下载对应的c ...
用python爬取高考网历年高考分数线将数据放入MySQL并绘制图表
用python爬取高考网历年高考分数线 # 导入爬虫的库 import requests from lxml import etree # 导入画图库 from pyecharts.charts im ...
爬取东方财富网股票行情数据和资讯
爬取东方财富网股票行情数据和资讯这个需求源于我的一个练手项目本篇博客参考:https://zhuanlan.zhihu.com/p/50099084 该博客介绍的东西本博客不做论述使用技术: 语 ...
利用python爬取东方财富网股吧评论并进行情感分析（一）
利用python爬取东方财富网股吧评论(一) python-东方财富网贴吧文本数据爬取分享一下写论文时爬数据用到的代码,有什么问题或者改善的建议的话小伙伴们一起评论区讨论.涉及内容在前人的研究基础之 ...

Python爬取晨星网的基金数据

如图：

CSV示例

如图：

Python爬取晨星网的基金数据相关推荐

最新文章

热门文章