python3 实现相似句语料生成-----爬取有道，google ,百度翻译生成

这里面涉及到 js加密的东西

具体可以参照 python 中execjs这个插件

直接上代码： https://gitee.com/zhoubingbing/google_translation

百度：

import execjs
import requests
class Baidu():@staticmethoddef create_en(text):url='https://fanyi.baidu.com/multitransapi'data={'from':'zh','to':'en','query':text,}re=requests.post(url,data=data)ls=re.json()return ls['data']['cands']@staticmethoddef create_cn(text):url='https://fanyi.baidu.com/v2transapi'with open('./fanyi/baidujs.js')as f:jsData=f.read()inputData=textdata={'query':inputData,"sign":execjs.compile(jsData).call("e",inputData),"token":"1d8992194dc3afd0fbb021f6e6633b1e",}headers={'Accept-Encoding':'gzip, deflate, br','Accept-Language':'zh-CN,zh;q=0.8','Connection':'keep-alive','Content-Length':'157','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8','Cookie':'BAIDUID=959BD58239197A4C8C27D9AA6CE6802A:FG=1; BIDUPSID=959BD58239197A4C8C27D9AA6CE6802A; PSTM=1537491023; REALTIME_TRANS_SWITCH=1; FANYI_WORD_SWITCH=1; HISTORY_SWITCH=1; SOUND_SPD_SWITCH=1; SOUND_PREFER_SWITCH=1; BDUSS=m5CZ3VNYWIxZWhzWkh6U2RTQ3Rockt0Z282U2huQTN-OXhkajZoRmhCYzFrTk5jSVFBQUFBJCQAAAAAAAAAAAEAAABW3BHTaHN6YjIyNTgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADUDrFw1A6xcY; DOUBLE_LANG_SWITCH=0; BDRCVFR[feWj1Vr5u3D]=I67x6TjHwwYf0; locale=zh; BCLID=11657211867648520518; BDSFRCVID=yIAOJeC62ABh8s59xJai2QKtlcL8MqvTH6aok6PAgEX5-VBnyVrgEG0Pex8g0KAbFpPcogKK0eOTHktF_2uxOjjg8UtVJeC6EG0P3J; H_BDCLCKID_SF=tbFJVILKtIvbfP0kKP7MM4LO5eT22-us32Qt2hcH0KLKVROXWlubeqDt-fJZLncGJKviBnrCJfb1MRjveb6WjtP3LqOwXpQy2gTDQl5TtUJaeCnTDMRhqt5W2b6yKMnitIT9-pnobpQrh459XP68bTkA5bjZKxtq3mkjbIOFfD_-MCL4j6L2en-W5gTtatctMCIX3buQt-cr8pcNLTDKj5K8hJ7NqRvZKRKHblv2aU5Wfnk90lO1j4_eKHouK4nZQC5v2nRJKhRi8h5jDh3q3jksD-Rt5t3byTry0hvctn5cShnc5MjrDRLbXU6BK5vPbNcZ0l8K3l02VKO_e6KbDjQQDN-Oq-cJHDv8QRoH5PK_Hn7zenrzLf4pbt-qJfoG2nbe_CTyWJjEhxjo5-vjX5kIKqjnBT5KaK0qbpjj3xPhKfQthRr4K4-kQN3TKRkO5bRi5RoI-UJjDn3oyT3qXp0nMpOly5jtMgOBBJ0yQ4b4OR5JjxonDhOyyG62btt_tbktoD5P; BDRCVFR[CZsEJLjIHxn]=mk3SLVN4HKm; delPer=0; PSINO=5; H_PS_PSSID=; BDORZ=FFFB88E999055A3F8A630C64834BD6D0; Hm_lvt_64ecd82404c51e03dc91cb9e8c025574=1555642094,1555642468,1555649103,1555655269; Hm_lpvt_64ecd82404c51e03dc91cb9e8c025574=1555655269; from_lang_often=%5B%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%2C%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%5D; to_lang_often=%5B%7B%22value%22%3A%22en%22%2C%22text%22%3A%22%u82F1%u8BED%22%7D%2C%7B%22value%22%3A%22zh%22%2C%22text%22%3A%22%u4E2D%u6587%22%7D%5D','Host': 'fanyi.baidu.com','Origin': 'https: // fanyi.baidu.com','Referer': 'https: // fanyi.baidu.com /?aldtype = 16047','User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0'}re=requests.post(url,data=data,headers=headers)re=re.json()return re['trans_result']['data'][0] ['dst']@staticmethoddef main(text,rest):re=Baidu.create_en(text)for i in re:rest.append(Baidu.create_cn(i))print('百度执行')print(rest)

google:

import json
from time import sleepimport requests
import execjs
import sys
class Google():def __init__(self):# self.proxies = self.get_proxies()self.proxies = ''self.headers = self.get_headers()def get_headers(self):headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0','Accept-Encoding': 'gzip, deflate','Accept': '*/*','Connection': 'keep-alive'}return headersdef set_proxies(self):url = 'http://d.jghttp.golangapi.com/getip?num=1&type=1&pro=&city=0&yys=0&port=1&time=3&ts=0&ys=0&cs=0&lb=1&sb=0&pb=4&mr=1&regions='response = requests.get(url)pro = response.text.strip()ip = pro.split(':')[0]port = pro.split(':')[1]proxies = {"http": "http://{}:{}".format(ip, port),"https": "https://{}:{}".format(ip, port),}return proxiesdef get_tk(self,text):ctx = execjs.compile("""function TL(a) {var k = "";var b = 406644;var b1 = 3293161072;var jd = ".";var $b = "+-a^+6";var Zb = "+-3^+b+-f";for (var e = [], f = 0, g = 0; g < a.length; g++) {var m = a.charCodeAt(g);128 > m ? e[f++] = m : (2048 > m ? e[f++] = m >> 6 | 192 : (55296 == (m & 64512) && g + 1 < a.length && 56320 == (a.charCodeAt(g + 1) & 64512) ? (m = 65536 + ((m & 1023) << 10) + (a.charCodeAt(++g) & 1023),e[f++] = m >> 18 | 240,e[f++] = m >> 12 & 63 | 128) : e[f++] = m >> 12 | 224,e[f++] = m >> 6 & 63 | 128),e[f++] = m & 63 | 128)}a = b;for (f = 0; f < e.length; f++) a += e[f],a = RL(a, $b);a = RL(a, Zb);a ^= b1 || 0;0 > a && (a = (a & 2147483647) + 2147483648);a %= 1E6;return a.toString() + jd + (a ^ b)};function RL(a, b) {var t = "a";var Yb = "+";for (var c = 0; c < b.length - 2; c += 3) {var d = b.charAt(c + 2),d = d >= t ? d.charCodeAt(0) - 87 : Number(d),d = b.charAt(c + 1) == Yb ? a >>> d: a << d;a = b.charAt(c) == Yb ? a + d & 4294967295 : a ^ d}return a}""")return ctx.call("TL", text)def c2e_url(self,text):baseUrl = 'https://translate.google.cn/translate_a/single?client=webapp'baseUrl += '&sl=zh-CN'baseUrl += '&tl=en'baseUrl += '&hl=zh-CN'baseUrl += '&dt=at'baseUrl += '&dt=bd'baseUrl += '&dt=ex'baseUrl += '&dt=ld'baseUrl += '&dt=md'baseUrl += '&dt=qca'baseUrl += '&dt=rw'baseUrl += '&dt=rm'baseUrl += '&dt=ss'baseUrl += '&dt=t'baseUrl += '&otf=1'baseUrl += '&ssel=3'baseUrl += '&tsel=3'baseUrl += '&kc=4'baseUrl += '&tk='+ self.get_tk(text)baseUrl += '&q='+textreturn baseUrldef e2c_url(self,text):baseUrl = 'https://translate.google.cn/translate_a/single?client=webapp'baseUrl += '&sl=en'baseUrl += '&tl=zh-CN'baseUrl += '&hl=zh-CN'baseUrl += '&dt=at'baseUrl += '&dt=bd'baseUrl += '&dt=ex'baseUrl += '&dt=ld'baseUrl += '&dt=md'baseUrl += '&dt=qca'baseUrl += '&dt=rw'baseUrl += '&dt=rm'baseUrl += '&dt=ss'baseUrl += '&dt=t'baseUrl += '&pc=1'baseUrl += '&otf=1'baseUrl += '&ssel=3'baseUrl += '&tsel=6'baseUrl += '&kc=1'baseUrl += '&tk=' + self.get_tk(text)baseUrl += '&q=' + textreturn baseUrldef set_similar(self,l):len_similar = len(l[5][0][2])all_similar = set()all_similar.add(l[0][0][0])for i in range(len_similar):all_similar.add(l[5][0][2][i][0])return all_similardef run(self,text,rest):sleep(0.5)all_chinese = set()all_chinese.add(text)try:englishs_response = requests.get(self.c2e_url(text),headers = self.headers,proxies =self.proxies,timeout=2)englishs = self.set_similar(json.loads(englishs_response.text))for i in englishs:chinese_response = requests.get(self.e2c_url(i),headers = self.headers,proxies =self.proxies,timeout=2)chineses = self.set_similar(json.loads(chinese_response.text))all_chinese.update(chineses)except:raise TimeoutErrorfor i in all_chinese:rest.append(i)print('google')print(rest)

有道：

import hashlib
import random
import time
import requests
class YouDao():def get_langue(self,text,FROM=None,TO=None):url='http://fanyi.youdao.com/bbk/translate_m.do'u = 'fanyideskweb'd = textf = str(int(time.time() * 1000) + random.randint(1, 10))c = '@6f#X3=cCuncYssPsuRUE'sign = hashlib.md5((u + d + f + c).encode('utf-8')).hexdigest()headers={'Accept':'application/json, text/javascript, */*; q=0.01','Accept-Encoding':'gzip, deflate','Accept-Language':'zh-CN,zh;q=0.8','Connection':'keep-alive','Content-Length':'221','Content-Type':'application/x-www-form-urlencoded; charset=UTF-8','Cookie':'OUTFOX_SEARCH_USER_ID=43732522@10.169.0.84; OUTFOX_SEARCH_USER_ID_NCOO=1140330123.9603255; SESSION_FROM_COOKIE=sogou-inc; JSESSIONID=abca3advC-8Hu_YSRbUOw; _ntes_nnid=3b993b2e5140076960c407074f2ceb2e,1555556922327; YOUDAO_FANYI_SELECTOR=OFF; DICT_UGC=be3af0da19b5c5e6aa4e17bd8d90b28a|; ___rl__test__cookies='+f,'Host':'fanyi.youdao.com','Origin':'http://fanyi.youdao.com','Referer':'http://fanyi.youdao.com/','User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.36 SE 2.X MetaSr 1.0','X-Requested-With':'XMLHttpRequest',}data={'i':d,'client':'fanyideskweb','salt':f,'sign':sign,'from':FROM,'to':TO,'doctype':'json','version':'3.0','cache':'true',}re=requests.post(url,data=data,headers=headers)results=re.json()results=results['translateResult']result=[]for i in results:result.append(i['tgt'].strip('\r\n'))return result@staticmethoddef main(text,rest):results= YouDao().get_langue(text, FROM='zh-CHS', TO='en')for i in results:rest.extend(YouDao().get_langue(i, FROM='en', TO='zh-CHS'))print('有道')print(rest)

import threading
from fanyi import baidu,google,youdao
if __name__=='__main__':while 1:rest=[]text=input('输入语料:')#三个线程if not text:break#谷歌t1=threading.Thread(target=google.Google().run,args=(text,rest))# 有道t3=threading.Thread(target=youdao.YouDao.main, args=(text,rest))#百度t2=threading.Thread(target=baidu.Baidu.main, args=(text,rest))t1.start()t2.start()t3.start()t1.join()t2.join()t3.join()print('混合：',list(set(rest)))

python3 实现相似句语料生成-----爬取有道，google ,百度翻译生成相关推荐

python 爬取有道词典的翻译 *渔夫版
前言在小甲鱼课程里学习到的python爬取有道词典的翻译,发现一些东西不是很适用于现在,网上给出的答案分为两种一.去掉 "_o" import urllib.request i ...
python爬取网页版QQ空间，生成词云图、柱状图、折线图（附源码）
python爬取网页版QQ空间,生成词云图.柱状图.折线图最近python课程学完了,琢磨着用python点什么东西,经过一番搜索,盯上了QQ空间,拿走不谢,欢迎点赞收藏,记得github给个sta ...
爬取qq音乐的评论并生成词云——以《听妈妈的话》为例
爬取qq音乐的评论并生成词云我们选取的是歌曲的周杰伦的听妈妈的话先看效果图首先,我们进去qq音乐找到这首歌网易云出来挨打 https://y.qq.com/n/yqq/song/002hXD ...
python利用mysql数据库实现一个中英文翻译程序兼单词试卷生成并改阅功能，并可以爬取有道官网进行在线翻译。
直接上运行截图(代码在后面) 开始界面使用navicat查看mysql数据库中的数据单词查询功能测试本地翻译一个本地数据库中有的单词本地翻译一个本地数据库中没有的单词爬取有道官网进行在线翻译 ...
python3爬虫系列16之多线程爬取汽车之家批量下载图片
python3爬虫系列16之多线程爬取汽车之家批量下载图片 1.前言上一篇呢,python3爬虫系列14之爬虫增速多线程,线程池,队列的用法(通俗易懂),主要介绍了线程,多线程,和两个线程池的使用. ...
python3[爬虫实战] 爬虫之requests爬取新浪微博京东客服
爬取的内容为京东客服的微博及评论思路:主要是通过手机端访问新浪微博的api接口,然后进行数据的筛选, 类似于这样的:https://m.weibo.cn/u/5650743478?uid=56507 ...
python爬取有道翻译
python爬虫爬取有道翻译教程编写环境为了宝宝们能够正确读懂本教程,在正式开始前,宝宝们需要搭建的环境如下: 连接互联网的win10电脑,(win7也可以) Google浏览器(版本无要求) P ...
python爬取有道词典json报错，求帮助！
python爬取有道词典json报错,求帮助! import urllib.request import urllib.parse import json import time import ran ...
python爬取百度使用kw关键字爬取时出现，百度安全验证，解决方法
python爬取百度使用kw关键字爬取时出现,百度安全验证,解决方法之前爬取百度用kw时的代码(没有任何问题) import requestsurl = 'http://www.baidu.com/ ...

python3 实现相似句语料生成-----爬取有道，google ,百度翻译生成

python3 实现相似句语料生成-----爬取有道，google ,百度翻译生成相关推荐

最新文章

热门文章