1 import urllib.error
 2 import urllib.request
 3 import urllib.parse
 4 import urllib
 5 import re
 6 import json
 7 import pandas as pd
 8 import time
 9 import logging
10 import random
11 from lxml import etree

1 import urllib.request
2
3 keyword = '上访'
4 once = urllib.request.quote(keyword)
5 pagecode = urllib.request.quote(once)
6
7 i = 1   # 页数
8 url = 'http://s.weibo.com/user/' + pagecode + '&page=' + str(i)

 1 data = urllib.request.urlopen(url,timeout=30).read().decode('utf-8')
 2
 3 lines = data.splitlines()
 4 for line in lines:
 5     if not line.startswith('<script>STK && STK.pageletM && STK.pageletM.view({"pid":"pl_user_feedList","js":'):
 6         continue
 7
 8     json_pattern = re.compile('\((.*)\)')
 9     # 利用正则取出json
10     json_data = json_pattern.search(line).group(1)
11     # 将json包装成字典并提取出html内容
12     html = json.loads(json_data)['html']

1 page = etree.HTML(html)
2 info_user['name'] = page.xpath('//a[@class="W_texta W_fb"]/@title')
3 info_user['_id'] = page.xpath('//a[@class="W_btn_b6 W_fr"]/@uid')
4 info_user['weibo_addr'] = page.xpath('//a[@class="W_texta W_fb"]/@href')
5 info_user['gender'] = page.xpath('//p[@class="person_addr"]/span[1]/@title')
6 info_user['location'] = page.xpath('//p[@class="person_addr"]/span[2]/text()')
7 info_user['follow'] = page.xpath('//p[@class="person_num"]/span[1]/a/text()')
8 info_user['fans'] = page.xpath('//p[@class="person_num"]/span[2]/a/text()')
9 info_user['weibo_num'] = page.xpath('//p[@class="person_num"]/span[3]/a/text()')

 1 # 提取简介信息
 2 def info(self, p, path):
 3     '''
 4     extract introduction of users
 5     :param p: input an etree
 6     :param path: input xpath which must be a string
 7     :return: a string
 8     '''
 9     if type(path) == str:
10         info = p.xpath(path)
11         if len(info) == 0:
12             sub_info = ''
13         else:
14             sub_info = info[0]
15
16         return sub_info
17     else:
18         print('please enter the path as a string')
19
20 # 提取标签信息：标签、教育信息、工作信息
21 def labels(self, p, path, key):
22     '''
23     extract labels, such as hobbits, education, job, of users
24     :param p: input an etree
25     :param path: input xpath which must be a string
26     :param key: keywords of labels
27     :return: a string
28     '''
29     label = p.xpath(path)
30     if len(label) == 0:
31         sub_label = ''
32     else:
33         for l in label:
34             label_name = re.compile('(.*?)：').findall(l.xpath('./text()')[0])
35             if label_name[0] == key:
36                 # 读取出标签信息下的所有标签内容
37                 all_label = l.xpath('./a/text()')
38                 l = ''
39                 for i in all_label:
40                     l = re.compile('\n\t(.*?)\n\t').findall(i)[0] + ',' + l
41                     sub_label = l
42             else:
43                 sub_label = ''
44
45     return sub_label

 1 info_user['intro'] = []
 2 info_user['标签'] = []
 3 info_user['职业信息'] = []
 4 info_user['教育信息'] = []
 5 others = page.xpath('//div[@class="person_detail"]')
 6 for p in others:
 7     path1 = './div/p/text()'
 8     info_user['intro'].append(self.info(p, path1))
 9     path2 = './p[@class="person_label"]'
10     info_user['标签'].append(self.labels(p, path2, '标签'))
11     info_user['职业信息'].append(self.labels(p, path2, '职业信息'))
12     info_user['教育信息'].append(self.labels(p, path2, '教育信息'))

 1 i = 1
 2 Flag = True
 3 while Flag:
 4     # 构建url
 5     url = 'http://s.weibo.com/user/' + pagecode + '&page=' + str(i)
 6     try:
 7         # 超时设置
 8         data = urllib.request.urlopen(url,timeout=30).read().decode('utf-8')
 9     except Exception as e:
10         print('出现异常 -->'+str(e))
11
12     …………
13
14     next_page = page.xpath('//a[@class="page next S_txt1 S_line1"]/@href')[0]
15     if len(next_page) == 0 :
16         Flag = False
17     else:
18         page_num = re.compile('page=(\d*)').findall(next_page)[0]
19         i = int(page_num)