山东大学创新项目实训个人工作日志（十七）

景点门票数据的整合和组织
主要实现的是景点门票的后四个平台（去哪儿、同程、途牛、携程）的数据的整合。
去哪儿：

    def search_spots(self, keyword, city):'''查询景点门票，核心方法:param keyword::param city::return:'''Ncity = city.replace('市', '').replace('县', '').replace('省', '')'''如果获取失败，则重试一次'''try:url = 'https://piao.qunar.com/ticket/list.htm?keyword='+Ncity+keyword+'&region='+self.getCityUrl(city)+'&from=mpl_search_suggest&page='except:try:url = 'https://piao.qunar.com/ticket/list.htm?keyword=' + Ncity + keyword + '&region=' + self.getCityUrl(city) + '&from=mpl_search_suggest&page='except:returnp = 1try:while (1):html = self.getHtml(url + str(p))soup = BeautifulSoup(html, "html.parser")div = soup.find_all('a', attrs={'data-click-type': 'l_title','class':'name'})if len(div) == 0:breakfor s in div:# 景点名称，门票（名称name，类别type，价格price，url，已售buy，旅行社from，可退isReturnable，预定时间bookTime,出票时间outTime，# 可用时间useTime,说明discription)title = s.stringresult = fuzz.token_sort_ratio(title, keyword)if result <= 20:continuehref = 'https://piao.qunar.com'+s['href']dis = []id = self.get_ticket_id(href)u = 'https://piao.qunar.com/ticket/detail/getTickets.json?sightId=455895&from=detail&supplierId='u = u.replace('455895', id)dicts = json.loads(self.getHtml(u))ls = dicts['data']['groups']slist = {}'''解析字典类型的数据'''for l in ls:if l['typeName']=='门票':infos = l['shelfVos']for info in infos:tickets = info['firstLvTicketTypeVos']for ticket in tickets:types = ticket['ticketTypes']for type in types:typeName = type['typeName']tis = type['tickets']for ti in tis:'''这是最小的一条门票的数据'''ttitle = ti['title']price = ti['qunarPrice']isreturn = ti['canRefundType']buyurl = ti['bookingUrl']fromw = '去哪儿网 '+ti['supplierName']booktime = ti['bookAtAnyTimeStr']dis = '''''数据合并'''slist.setdefault(typeName, [])slist[typeName].append({'name': ttitle, 'type': typeName, 'price': price, 'url':buyurl,'buy': '', 'from': fromw, 'isReturnable': isreturn,'bookTime': booktime, 'outTime': '', 'useTime': '','discription': dis})self.spotsInfo[title] = slist# print(self.spotsInfo)# self.tickets.append(dis)# self.spots.append([title,href])p = p + 1except:traceback.print_exc()self.done = True# print(self.tickets)# print(self.spots)

同程：

    def search_spots(self, keyword, city):''':param keyword: 关键词:param city: 在哪所城市搜索:return:'''Ncity = city.replace('市', '').replace('县', '').replace('省', '')url = 'https://so.ly.com/scenery?q='+Ncity+keywordp = 1try:html = self.getHtml(url)soup = BeautifulSoup(html,"html.parser")divs = soup.find_all('div',{'class':'list_l'})for div in divs:sid = div.find('div',{'class':'s_info'})['sid']title = div.find('a',{'class':'sce_name goFinal'})['title']'''筛选无关信息'''result = fuzz.token_sort_ratio(title, keyword)if result <= 20:continuedetailurl = 'https://so.ly.com/scenery/AjaxHelper/SceneryPriceFrame.aspx?action=GETNEWFRAMEFORLIST&showCount=2&ids='+str(sid)+'&isSimple=1&priceList=1&tabself=1&tabHotel=1&isGrap=1&nobookid=&isyry=1&YpState=1&lon=0&lat=0&isforegin=0&isNewSearch=true&iid=0.398138641670454'detailhtml = self.getHtml(detailurl)ticketInfo = json.loads(detailhtml)buyurl = 'https://www.ly.com/scenery/BookSceneryTicket_'+str(sid)+'.html'SceneryPrices = ticketInfo['SceneryPrices']# print(title)slist = {}for SceneryPrice in SceneryPrices:types = SceneryPrice['ChannelPriceModelEntityList']for type in types:typename = type['ConsumersTypeName']ChannelPriceEntityList = type['ChannelPriceEntityList']for ChannelPriceEntity in ChannelPriceEntityList:ttitle = ChannelPriceEntity['TicketName']price = ChannelPriceEntity['AmountAdvice']'''门票详细描述'''dis = '入园方式：\n'+ChannelPriceEntity['GetTicketMode']dis = dis + '预订说明：\n'+parse.unquote(ChannelPriceEntity['PriceBookRemark'])dis = dis + '预订时间：\n'+ChannelPriceEntity['PriceTimeLimit']dis = dis+'包含项目：\n'+ChannelPriceEntity['ContainedItems']dis = dis+'退改规则：\n'+ChannelPriceEntity['RefundModifyRule']ttypename = ChannelPriceEntity['ConsumersTypeName']booktime = ChannelPriceEntity['BookTime']dis = dis+'商家信息：\n'+ChannelPriceEntity['SupplierBaseInfo']['SupplierName']'''数据合并'''slist.setdefault(ttypename, [])slist[ttypename].append({'name': ttitle, 'type': ttypename, 'price': price, 'url': buyurl,'buy': '', 'from': '同程旅游', 'isReturnable': '','bookTime': booktime, 'outTime': '', 'useTime': '','discription': dis})self.spotsInfo[title] = slist# print(self.spotsInfo)# print(price)except Exception as e:print(e)self.done = True

途牛：

    def search_spots(self, keyword, city):'''核心方法:param keyword: 关键词:param city: 城市:return:'''try:cityUrl = self.getCityUrl(city)except:'''失败则重试一次'''try:cityUrl = self.getCityUrl(city)except:returnNcity = city.replace('市','').replace('县','').replace('省','')url = 'https://s.tuniu.com/search_complex/ticket-'+cityUrl+'-0-'+Ncity+keyword+'/'p = 1;try:while (1):html = self.getHtml(url + str(p))soup = BeautifulSoup(html, "html.parser")div = soup.find('div', {'class': 'thelist'})if div is None:breakspot = div.find_all('li')for s in spot:slist = {}title = s.find('dl', {'class': 'detail'}).find('p',{'class':'title ticket'}).text.replace('\n','').replace(' ','')result = fuzz.token_sort_ratio(title, keyword)if result <= 20:continue'''景点地址'''address = s.find('dl', {'class': 'detail'}).find('dd').string.replace(u'\u3000',u'')'''开放时间'''opentime = s.find('dl', {'class': 'detail'}).find('dd', {'class': 'port'}).text.replace('\n','')price = s.find('div', {'class': 'priceinfo'}).find('span',{'class':'tnPrice'}).text.replace('\n','').replace(' ','')'''满意度'''manyi = s.find('p',{'class': 'manyi_inner'}).text.replace('\n','').replace(' ','')'''景点链接'''href = 'https:'+(s.find('div',{'class':'theinfo ticket clearfix'}).find('a',{'class':'img'}))['href']'''景点图片链接'''imgsrc = 'https:'+(s.find('div',{'class':'theinfo ticket clearfix'}).find('img'))['data-src']ticket = s.find('div',{'class': 'ticketlist'}).find_all('div',{'class': 'each-item'})dis = []for t in ticket:'''景点门票处理'''ticketTitle = t.find('span', {'class': 'ticketTitle'}).textticketPrice = t.find('span', {'class': 'tnPrice'}).textticketBookUrl = (t.find('form'))['action']slist.setdefault('', [])slist[''].append({'name': ticketTitle, 'type': '', 'price': ticketPrice, 'url': 'https:'+ticketBookUrl,'buy': '', 'from': '途牛旅游', 'isReturnable': '','bookTime': '', 'outTime': '', 'useTime': '','discription': ''})dis.append([ticketTitle,ticketPrice,ticketBookUrl])# self.tickets.append(dis)self.spotsInfo[title] = slistself.spots[title] = {'title':title,'address':address,'opentime':opentime,'manyi':manyi,'href':href,'imgsrc':imgsrc}p = p + 1except:pass# print(self.spotsInfo)# print(self.spots)self.done = True

携程：

    def get_ticket(self):'''获取景点门票信息:return:'''id = self.detail_url.split('/')[-1]# print(id)ticket_url = f'https://piao.ctrip.com/ticket/dest/{id}?onlyContent=true&onlyShelf=true'# print(ticket_url)ticket_res = requests.get(ticket_url, verify=False, headers=self.headers).text# time.sleep(1)ticket_res = ticket_res.replace('\n','').replace(' ','')ticket_res = ticket_res[ticket_res.find('window.__INITIAL_STATE__')+25:ticket_res.find('window.__APP_SETTINGS__')]info = json.loads(ticket_res)ticketinfos = info['detailInfo']['ressHash']slist = {}for ticketinfo in ticketinfos.values():'''解析字典数据'''title = ticketinfo['name']price = ticketinfo['price']type = ticketinfo['saleunitinfo']['propleproperty']fromw = '携程旅游 '+ticketinfo['brandname']'''数据合并'''slist.setdefault(type, [])slist[type].append({'name': title, 'type': type, 'price': price, 'url': self.detail_url,'buy': '', 'from': fromw, 'isReturnable': '','bookTime': '', 'outTime': '', 'useTime': '','discription': ''})self.spotsInfo[self.title] = slist

八个平台的数据合并：

def merge2(spi, xc, name):'''把两个字典数据合并在一起:param spi::param xc::param name::return:'''if len(name) == 0:spotsInfo = spielse:spotsInfo = {}for info in spi.keys():try:spotsInfo.setdefault(info, {})k = xc.spotsInfo[info]if len(name) == 0:spotsInfo[info][xc.name] = xc.spotsInfo[info]else:spotsInfo[info][name] = spi[info]spotsInfo[info][xc.name] = xc.spotsInfo[info]xc.spotsInfo.pop(info)except Exception as e:# print(e)if len(name) != 0:spotsInfo[info][name] = spi[info]for info in xc.spotsInfo.keys():spotsInfo.setdefault(info, {})spotsInfo[info][xc.name] = xc.spotsInfo[info]# print(spotsInfo)print('done')return spotsInfodef merge(feizhu,xiecheng,tuniu,quna,lvmama,dahe,klook,tongcheng):'''把所有的平台的数据都合并在一起:param feizhu::param xiecheng::param tuniu::param quna::param lvmama::param dahe::param klook::param tongcheng::return:'''list = [feizhu,xiecheng,tuniu,quna,lvmama,dahe,klook,tongcheng]merged = FalsespotsInfo = {}while len(list)>0:l = list[0]if merged:spotsInfo = merge2(spotsInfo,list.pop(0),'')else:spotsInfo = merge2(list.pop(0).spotsInfo,list.pop(0),l.name)merged = True# print(spotsInfo)return spotsInfo

山东大学创新项目实训个人工作日志（十七）相关推荐

山东大学创新项目实训个人工作日志（一）
目录一.百度地图API的接入二.主要实现的功能查询景点计算距离半正矢公式一.百度地图API的接入百度地图API可以为我们提供多种关于地图绘制和计算的功能,因为我们的项目和出行旅游有着很大 ...
山东大学创新项目实训个人工作日志（二十三）
景点图片爬取(改进) 经过测试发现,之前爬到的数据并不是全部的数据,据估计,全国共两千多个城市和地区,数以万计的景点,而之前仅仅是将每个城市的一小部分景点爬了出来,所以现在需要做的就是把所有数据全部爬 ...
山东大学创新项目实训个人工作日志（十六）
景点门票数据的整合至此为止,共爬取了八个平台上门票信息,但信息参差不齐,如果直接这样返回给用户肯定是不可以的,所以现在需要做的就是把八个平台上想要爬取的数据规定一个固定的格式,我这里采用的依然是字典 ...
山东大学创新项目实训个人工作日志（二十）
马蜂窝平台的景点图片爬取这次完成的是对马蜂窝景点图片爬取的改进,之前是通过地图点来爬取的,但是地图点的数据不全,所以研究之后打算从这一板块重新爬取. 但是这一部分进行翻页时,网页本身是没有任何变化的 ...
山东大学软件学院项目实训-创新实训-SDUMeeting(一)
山东大学软件学院项目实训-创新实训-SDUMeeting(一) 一.前言: 这个项目是山东大学视频会议项目,这个项目基于webrtc构建多人视频会议系统,我负责视频会议客户端及服务端安全防护,这个专栏 ...
山东大学软件学院项目实训-创新实训-SDUMeeting(三)
山东大学软件学院项目实训-创新实训-SDUMeeting(三) 山大会议项目web安全之文件注入防护一.前言山大会议的客户端加入了对用户自定义头像的支持,需要用户自行上传头像图片,这个过程存在文件 ...
山东大学软件学院项目实训-创新实训-SDUMeeting(六)
山东大学软件学院项目实训-创新实训-SDUMeeting(六) 端对端加密与密钥交换端到端加密(end-to-end),是一种只有参与通讯的用户可以读取信息的通信系统.它可以防止潜在的窃听者--包括 ...
山东大学暑期项目实训——云主机服务比价与预测系统
山东大学暑期项目实训--云主机服务比价与预测系统(十二) 使用tensorflow做预测一.简单的预测函数值 1.使用训练数据做预测创建模型: model = keras.Sequential([ ...
山东大学软件学院项目实训-创新实训-网络安全靶场实验平台（一）
目录前言一.项目介绍二.项目框架三.个人分工四.本周任务完成情况 4.1 Vue安装 4.1.1 node.js安装及配置 4.1.2 安装vue及脚手架 4.1.3 运行Vue项目 4. ...

山东大学创新项目实训个人工作日志（十七）

山东大学创新项目实训个人工作日志（十七）相关推荐

最新文章

热门文章