import requests
import re
from lxml import etree
from aqicn import crawldef get_cities():headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.92 Safari/537.36'}content = requests.get('http://aqicn.org/city/all/cn/',headers=headers)response = content.content.decode('utf-8')hongkong = re.compile(r'id=\'香港\'></a>(.*)<a id=\'蒙古\'></a>',re.S)taiwan = re.compile(r'id=\'台湾\'></a>(.*)<a id=\'新加坡\'></a>',re.S)land = re.compile(r'id=\'中国\'></a>(.*)<a id=\'越南\'></a>',re.S)hongkong = hongkong.findall(response)[0]hongkong = etree.HTML(hongkong)taiwan = taiwan.findall(response)[0]taiwan = etree.HTML(taiwan)land = land.findall(response)[0]land = etree.HTML(land)city_list = land.xpath('//a') + hongkong.xpath('//a') + taiwan.xpath('//a')for i in city_list:try:url = i.xpath('.//@href')[0]name = i.xpath('.//text()')[0].strip()if name:yield {'url':url,'city':name}except Exception as e:print(e)def task_manager():for data in get_cities():crawl.delay(data)if __name__ == '__main__':task_manager()

import requests
from lxml import etree
from celery import Celery# 这里定义了broker和backend
# 注意IP和后面的数字都是可以调整的
app = Celery('aqicn', broker='redis://192.168.4.53:6379/1', backend='redis://192.168.4.53:6379/3')# 装饰器，说明这是一个task的实现
@app.task
def crawl(data):headers = {'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8','Accept-Encoding': 'gzip, deflate','Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8','Host': 'aqicn.org','Proxy-Connection': 'keep-alive','Referer': 'http://aqicn.org/city/all/cn/','Upgrade-Insecure-Requests': '1','User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36',}url = data['url']response = requests.get(url, headers=headers)html = response.content.decode('utf-8')html = etree.HTML(html)try:aqi = html.xpath('//div[@id="aqiwgtvalue"]//text()')[0] # 空气质量except:news = html.xpath('//div[@class="section-content"]/center//h3//text()')if news:aqi= ''.join(news)else:aqi = '页面加载错误'data['aqi'] = aqireturn data

import redis, picklerediscli = redis.StrictRedis(host='192.168.4.53', db=3, port=6379)while True:result = rediscli.keys() # 持续从redis里拿keysfor i in result:key = i.decode('utf-8')data = rediscli.get(key)try:print(pickle.loads(data))rediscli.delete(key)except Exception as e:print(e)