一、设计UI页面

一些组件需要认识一下，文本框、按钮、进度条等。

对于UI这些不明白的参考上篇教程，第一个pyqt5程序

二、UI文件生成PY文件

1.快捷生成py文件

py文件会自动生成在当前目录下，与ui文件同名

2.运行一下这个py页面

导入的文件名为你生成的py文件名称

注意包名

import elk.txtToExcel as txtToExcel
#你们的应该是这样就可以了
import txtToExcel

import sys
import elk.txtToExcel as txtToExcel
from PyQt5.QtWidgets import QApplication,QMainWindowif __name__ == '__main__':app=QApplication(sys.argv)mainWindow=QMainWindow()ui=txtToExcel.Ui_MainWindow()# 向主窗口添加控件ui.setupUi(mainWindow)mainWindow.show()sys.exit(app.exec_())

如果你的import导入有问题，请修改你项目下代码包的设置

3.编码

在这之前还要说一下objectName属性，它相当于控件对象的名字。你要操作这个控件，可以使用这个名字来调用对应的方法。

开整，首先需要把Object改成QWidget。

import re
import tkinter as tk
from tkinter import filedialog
from PyQt5 import QtCore, QtWidgets
from PyQt5.QtWidgets import *
import os
import xlrd
from openpyxl import Workbook as owb
import json
import requests
from datetime import datetime
import time
import datetime
#   这里需要改一下 ↓↓↓
class Ui_MainWindow(QWidget):

先拿一个最简单的例子来说，给退出按钮实现关闭的功能。
在刚刚生成好的py文件中，有一个全是初始化你那些控件的方法找到它。

在里边增加这么一行代码。
clicked点击
connect执行对应的函数
MainWindow.close关闭窗口函数

self.exitBtn.clicked.connect(MainWindow.close)

按照上边的例子，需要你自己实现包括开始和其他按钮的实现逻辑。
我就放在了第二个方法里了。

 def retranslateUi(self, MainWindow):_translate = QtCore.QCoreApplication.translateMainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))self.actionBtn.setText(_translate("MainWindow", "开始"))self.exitBtn.setText(_translate("MainWindow", "退出"))self.label.setText(_translate("MainWindow", "excel所在路径："))self.excelBtn.setText(_translate("MainWindow", "浏览"))self.label_2.setText(_translate("MainWindow", "设置保存位置："))self.saveBtn.setText(_translate("MainWindow", "浏览"))self.label_3.setText(_translate("MainWindow", "当前进度："))#设置进度条初始为0%self.numBar.setValue(0)#对应按钮对应的函数self.excelBtn.clicked.connect(self.setTargetPath)self.saveBtn.clicked.connect(self.setSavePath)self.actionBtn.clicked.connect(self.start)self.thread = None  # 初始化线程# 主函数def start(self):pass# 设置目标文件夹路径def setTargetPath(self):# 隐藏控制台root = tk.Tk()root.withdraw()self.targetPath = filedialog.askdirectory()self.excelText.setText(self.targetPath)# 查找文件夹下所有的文件def findAllFile(self, base):for root, ds, fs in os.walk(base):for f in fs:yield f# 设置保存文件夹路径def setSavePath(self):# 隐藏控制台root = tk.Tk()root.withdraw()self.savePath = filedialog.askdirectory()self.saveText.setText(self.savePath)

在主函数，也就是start这个函数中，我们需要开一个子线程去进行爬取和写入excel操作。并且实时对进度条进行更。
主线程需要传递给子线程两个参数方法如下：

 # 主函数def start(self):# 创建线程self.actionBtn.setEnabled(False)# 执行工作的子线程self.thread = WorkThread(fileList=filenamelist,savePath = self.savePath)# 连接信号self.thread._signal.connect(self.call_backlog)  # 进程连接回传到GUI的事件# 开始线程self.thread.start()

其中的WorkThread是一个具体执行爬虫和写入excel的类，继承QThread。

对应子线程中构造方法中的两个参数，用来主线程和子线程进行传参
WorkThread(fileList=filenamelist,savePath = self.savePath)

以下均为同一个类

from PyQt5.QtCore import *
from PyQt5 import QtCore
import xlrd
from openpyxl import Workbook as owb
import json
import requests
from datetime import datetime
import time
import datetime
class WorkThread(QtCore.QThread):#  通过类成员对象定义信号对象_signal = pyqtSignal(int)#接收主线程传参fileList，savePathdef __init__(self,fileList,savePath):super(WorkThread, self).__init__()self.fileList = fileListself.savePath = savePathdef __del__(self):self.wait()

    # 具体执行任务的方法def run(self):for i in self.fileList:# 使用openexcel_file = owb()# 在Excel文件里创建一个工作表sheet_nameexcel_table = excel_file.create_sheet('收集结果', 0)fileName = str(i).replace('.xls', '').replace('.xlsx', '').split('/')[-1]xlrds = xlrd.open_workbook(i)# 打开指定的表table = xlrds.sheet_by_index(0)#开始执行收集的方法self.startCollect(table, excel_file, fileName, excel_table, self.savePath)# 设置进度条为100%self._signal.emit(100)

收集方法，涉及到动态改变进度条self._signal.emit(((item/(rows+1)))*100)。

    def startCollect(self, table, excel_file, fileName, excel_table, folderPath):savePath = folderPath + '/提取的trace信息文件夹'# 获取总行数rows = table.nrowsexcel_table.cell(1, 1, 'message:')excel_table.cell(1, 2, 'trice:')excel_table.cell(1, 3, 'uri:')excel_table.cell(1, 4, 'mode:')excel_table.cell(1, 5, 'clientVersion:')excel_table.cell(1, 6, 'loginType:')excel_table.cell(1, 7, 'serverVersion:')index_line = 2for item in range(0, rows):#动态改变进度条self._signal.emit(((item/(rows+1)))*100)tid = "TID:" + str(table.cell(item, 0).value)#爬虫及写入excel方法self.startChild(tid, index_line, excel_table, table)index_line += 1# 保存文档excel_file.save(savePath + '/' + fileName + '完成' + '.xlsx')

具体实现逻辑，你使用Discover - Elastic是会有请求参数，但你按一条traceid进行请求时。每次只会traceid不一样。所以这里就只动态改变了tid和筛选时间范围。若果增加条件只需要自己重新发一次请求加到body里即可。
这里的条件在网页是涨这个样子，怎加了一个错误级别。并且时间倒序。然后后边解析返回的json对象去第一条报错信息。

trace: "TID:4c692741e59a452cbc8b4d5de06b8837.75.16419144765607809" and level : "ERROR"

注意！！！我这个body是不全的，把一些动态的我加上了，涉及到公司一下东西所以没有放上去，需要你们自己先把条件设置好请求一遍在粘贴到body里，然后把需要动态的地方替换掉即可

    def startChild(self, tid, row, excel_table, table):body = {"params": {"ignoreThrottled": True,"index": "","body": {"version": True,"size": 500,"sort": [{"@timestamp": {"order": "asc","unmapped_type": "boolean"}}],"query": {"bool": {"must": [],"filter": [{"bool": {"should": [{"match_phrase": {"trace": tid}}],"minimum_should_match": 1}},{"bool": {"should": [{"match_phrase": {"level": "ERROR"}}],"minimum_should_match": 1}},{"range": {"@timestamp": {"gte": datetime.datetime.utcfromtimestamp(time.mktime((datetime.datetime.now() - datetime.timedelta(days=20)).timetuple())).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),"lte": datetime.datetime.utcfromtimestamp(time.mktime((datetime.datetime.now() + datetime.timedelta(days=1)).timetuple())).strftime("%Y-%m-%dT%H:%M:%S.%fZ"),"format": "strict_date_optional_time"}}}],"should": [],"must_not": []}}}...}}jsonData = json.dumps(body)headers = {#需要你自己填写请求头}response = requests.post(url='你们的请求url地址', data=jsonData,headers=headers, timeout=10)response.encoding = 'utf-8'result = response.json()try:if str(result['rawResponse']['hits']['hits'][0]['_source']['ex_detail']) == '' or str(result['rawResponse']['hits']['hits'][0]['_source']['ex_detail']) is None:excel_table.cell(row, 1, str(result['rawResponse']['hits']['hits'][0]['_source']['message']))excel_table.cell(row, 2, str(result['rawResponse']['hits']['hits'][0]['_source']['trace']))# 处理uriuri = str(result['rawResponse']['hits']['hits'][0]['_source']['uri'])uri.replace("POST", "").replace("GET", "")# uri = re.sub(r'\s+', "", uri).replace("http://", "")# uri = re.sub(r'\d', "", uri).replace("GET", "")# uri = re.sub(r'\.(.*):', "", uri).replace("POST", "")if uri == '' or uri is None or uri == 'null':uri = '无uri地址'excel_table.cell(row, 3, uri)excel_table.cell(row, 4, str(result['rawResponse']['hits']['hits'][0]['_source']['service']))clientVersion = str(result['rawResponse']['hits']['hits'][0]['_source']['clientVersion'])if clientVersion == '' or clientVersion is None or clientVersion == 'null':clientVersion = '无clientVersion'excel_table.cell(row, 5, clientVersion)loginType = str(result['rawResponse']['hits']['hits'][0]['_source']['loginType'])if loginType == '' or loginType is None or loginType == 'null':loginType = '无loginType'excel_table.cell(row, 6, loginType)serverVersion = str(result['rawResponse']['hits']['hits'][0]['_source']['serverVersion'])if serverVersion == '' or serverVersion is None or serverVersion == 'null':serverVersion = '无serverVersion'excel_table.cell(row, 7, serverVersion)else:excel_table.cell(row, 1, str(result['rawResponse']['hits']['hits'][0]['_source']['ex_detail']))excel_table.cell(row, 2, str(result['rawResponse']['hits']['hits'][0]['_source']['trace']))# 处理uriuri = str(result['rawResponse']['hits']['hits'][0]['_source']['uri'])uri.replace("POST", "").replace("GET", "")# uri = re.sub(r'\s+', "", uri).replace("http://", "")# uri = re.sub(r'\d', "", uri).replace("GET", "")# uri = re.sub(r'\.(.*):', "", uri).replace("POST", "")if uri == '' or uri is None or uri == 'null':uri = '无uri地址'excel_table.cell(row, 3, uri)excel_table.cell(row, 4, str(result['rawResponse']['hits']['hits'][0]['_source']['service']))clientVersion = str(result['rawResponse']['hits']['hits'][0]['_source']['clientVersion'])if clientVersion == '' or clientVersion is None or clientVersion == 'null':clientVersion = '无clientVersion'excel_table.cell(row, 5, clientVersion)loginType = str(result['rawResponse']['hits']['hits'][0]['_source']['loginType'])if loginType == '' or loginType is None or loginType == 'null':loginType = '无loginType'excel_table.cell(row, 6, loginType)serverVersion = str(result['rawResponse']['hits']['hits'][0]['_source']['serverVersion'])if serverVersion == '' or serverVersion is None or serverVersion == 'null':serverVersion = '无serverVersion'excel_table.cell(row, 7, serverVersion)except Exception as e:flage = Truefor con in range(0, 4):try:self.errorDel(result['id'], row, excel_table)flage = Truebreakexcept Exception as e:flage = False# print(e)continueif not flage:print('4次请求都失败了')excel_table.cell(row, 2, str(table.cell(row, 0).value))excel_table.cell(row, 1, 'tid获取信息异常')#如果获取失败进行二次获取的方法def errorDel(self, id, row, excel_table):body = {"id": str(id)}jsonData = json.dumps(body)headers = {#请求头信息}response = requests.post(url='请求地址', data=jsonData,headers=headers, timeout=10)result = response.json()if str(result['rawResponse']['hits']['hits'][0]['_source']['ex_detail']) == '' or str(result['rawResponse']['hits']['hits'][0]['_source']['ex_detail']) is None:excel_table.cell(row, 1, str(result['rawResponse']['hits']['hits'][0]['_source']['message']))excel_table.cell(row, 2, str(result['rawResponse']['hits']['hits'][0]['_source']['trace']))# 处理uriuri = str(result['rawResponse']['hits']['hits'][0]['_source']['uri'])# uri = re.sub(r'\s+', "", uri).replace("http://", "")# uri = re.sub(r'\d', "", uri).replace("GET", "")# uri = re.sub(r'\.(.*):', "", uri).replace("POST", "")uri.replace("POST", "").replace("GET", "")if uri == '' or uri is None or uri == 'null':uri = '无uri地址'excel_table.cell(row, 3, uri)excel_table.cell(row, 4, str(result['rawResponse']['hits']['hits'][0]['_source']['service']))clientVersion = str(result['rawResponse']['hits']['hits'][0]['_source']['clientVersion'])if clientVersion == '' or clientVersion is None or clientVersion == 'null':clientVersion = '无clientVersion'excel_table.cell(row, 5, clientVersion)loginType = str(result['rawResponse']['hits']['hits'][0]['_source']['loginType'])if loginType == '' or loginType is None or loginType == 'null':loginType = '无loginType'excel_table.cell(row, 6, loginType)serverVersion = str(result['rawResponse']['hits']['hits'][0]['_source']['serverVersion'])if serverVersion == '' or serverVersion is None or serverVersion == 'null':serverVersion = '无serverVersion'excel_table.cell(row, 7, serverVersion)else:excel_table.cell(row, 1, str(result['rawResponse']['hits']['hits'][0]['_source']['ex_detail']))excel_table.cell(row, 2, str(result['rawResponse']['hits']['hits'][0]['_source']['trace']))# 处理uriuri = str(result['rawResponse']['hits']['hits'][0]['_source']['uri'])# uri = re.sub(r'\s+', "", uri).replace("http://", "")# uri = re.sub(r'\d', "", uri).replace("GET", "")# uri = re.sub(r'\.(.*):', "", uri).replace("POST", "")uri.replace("POST", "").replace("GET", "")if uri == '' or uri is None or uri == 'null':uri = '无uri地址'excel_table.cell(row, 3, uri)excel_table.cell(row, 4, str(result['rawResponse']['hits']['hits'][0]['_source']['service']))clientVersion = str(result['rawResponse']['hits']['hits'][0]['_source']['clientVersion'])if clientVersion == '' or clientVersion is None or clientVersion == 'null':clientVersion = '无clientVersion'excel_table.cell(row, 5, clientVersion)loginType = str(result['rawResponse']['hits']['hits'][0]['_source']['loginType'])if loginType == '' or loginType is None or loginType == 'null':loginType = '无loginType'excel_table.cell(row, 6, loginType)serverVersion = str(result['rawResponse']['hits']['hits'][0]['_source']['serverVersion'])if serverVersion == '' or serverVersion is None or serverVersion == 'null':serverVersion = '无serverVersion'excel_table.cell(row, 7, serverVersion)

Discover - Elastic通过trace抓取信息到EXCEL相关推荐

《精通Wireshark》—第1章1.5节抓取信息的方式
本节书摘来自异步社区<精通Wireshark>一书中的第1章1.5节抓取信息的方式,作者[印度]Charit Mishra(夏里特米什拉),更多章节内容可以访问云栖社区"异步社 ...
PHP PDF内容识别抓取信息方法
PHP PDF内容识别抓取信息方法 PDF Parser 使用 PDF Parser 参考:http://www.pdfparser.org/ (注意:composer.json 更新 pdfpa ...
python抓取网页内容到excel_Python实现抓取网页生成Excel文件的方法示例
本文实例讲述了Python实现抓取网页生成Excel文件的方法.分享给大家供大家参考,具体如下: Python抓网页,主要用到了PyQuery,这个跟jQuery用法一样,超级给力示例代码如下: # ...
uibot数据抓取+数据表+excel小案例
uibot数据抓取+数据表+excel小案例 Dim arrayData = "" Dim objDatatable = "" Dim objExcelWork ...
python统计自己微信好友并抓取信息
前几天统计自己好友性别,看看男女比例,发现竟然还有分类不是男女的,很好奇都是谁,所以空闲下来抓取所有好友看一下. 这边使用了itchat库,网上资料很多.不多说,直接上代码 import itchat ...
VUE制作谷歌浏览器多类型截屏、抓取信息的插件
提到VUE大家都很熟悉,VUE可以做很多东西,包括SPA单页面应用,WEBAPP应用,小程序应用等等,但是你们有没有考虑过过,使用VUE来制作一款抓取网页数据(文字+截屏)的扩展呢?敢想就要敢做,Le ...
php爬虫抓取信息及反爬虫相关
php爬虫首推Curl函数了,先来认识下它. 0x01.curl扩展的安装: 1.确保php子文件夹ext里面有php_curl.dll(一般都有的,一般配置时候会设置环境变量的) 2.将php.in ...
关于网页抓取信息 Jsoup 和 HttpUnit 的使用
之前看了看洪祥的csdn app 制作,里面有从网页抓取数据的. 之前对这块操作没有接触过,所以网上差了一些资料,现在整理下相关的内容. 首先是关于Jsoup , Jsoup 是网页抓取的首选.但是 ...
python爬虫抓取信息_python爬虫爬取网上药品信息并且存入数据库
我最近在学习python爬虫,然后正好碰上数据库课设,我就选了一个连锁药店的,所以就把网上的药品信息爬取了下来. 1,首先分析网页 2,我想要的是评论数比较多的,毕竟好东西大概是买的人多才好.然后你会 ...

Discover - Elastic通过trace抓取信息到EXCEL

文章目录