课程链接

讲师的公众号文章：今日头条数据抓取及持久化(完整代码版,含IP和用户代理)​mp.weixin.qq.com

课程代码

抓取并持久化user-agent工具utils.py

对于爬虫工具，需要设置发起请求的user-agent，import fake-useragent可以实现，但是每次更换user-agent都要发起网络请求，并不是好办法，所以比较好的做法是：抓取文件，本地json存储备用。

怎么找到fake-useragent到哪里取数的，截图如下：

utils.py代码如下：

"""

这个工具的目的是把fake_useragent的user_agent下载下来，存储文件使用

from fake_useragent import UserAgent

https://fake-useragent.herokuapp.com/browsers/0.1.11

"""

import requests

import json

import random

"""

json的方法中 dump,dumps,load,loads的区别

dump/load，是操作文件的

dumps/loads，是处理json和字符串直接相互转换的

作业：拿到标签并存储起来

"""

browser_info_file = "browser_info.json"

user_agent_url = "https://fake-useragent.herokuapp.com/browsers/0.1.11"

def write_browser_info_to_file():

my_user_agent = requests.get(user_agent_url)

with open(browser_info_file, "w") as f:

# 写进去就是JSON，方便用json.load读取

json.dump(my_user_agent.text, f)

# f.write 的写入方式，是个字符串，json.load就会有问题

# f.write(my_user_agent.text)

def get_random_browser():

file = open(browser_info_file, 'r')

#

agent_json = json.load(file)

#

agent_json = json.loads(agent_json)

browsers = agent_json["browsers"]

# 随机出来一个浏览器类型

i = random.randint(0, len(browsers) - 1)

if i == 0:

browser_name = "chrome"

elif i == 1:

browser_name = "opera"

elif i == 2:

browser_name = "internetexplorer"

else:

browser_name = "safari"

final_browser = browsers[browser_name][random.randint(0, len(browsers[browser_name]))]

return final_browser

if __name__ == '__main__':

write_browser_info_to_file()

print(get_random_browser())

抓取热门文章tags并存储-toutiao_spider.py

import requests

import time

import json

import pandas as pd

import re

from part1.utils import get_random_browser

# 定义Http请求的headers/proxies

def get_request_url_and_headers():

user_agent = get_random_browser()

current_time = int(time.time())

# 我要到哪里去

# 头条的这个，去掉：_signature，否则抓不到数

request_url = "https://www.toutiao.com/api/pc/feed/?max_behot_time=" \

+ str(current_time) \

+ "&category=news_hot&utm_source=toutiao&widen=1&tadrequire=true&as=A1751D55DE389D7&cp" \

"=5D5E38E9CDA7BE1"

# 我怎么来，我骑着自行车来

headers = {

"user-agent": user_agent

}

# 我从哪里来，我从东土大唐而来

# 搜索引擎搜"免费IP地址"能找到可用的代理

proxies = {

"url": "http://114.235.23.172:9000"

}

return request_url, headers, proxies

# 发起Http请求，抓取网页内容，如果发生错误，重试一次

def get_response_json():

request_url, headers, proxies = get_request_url_and_headers()

response = requests.get(request_url, headers=headers, proxies=proxies)

# print(response.text.encode("utf-8").decode("unicode_escape"))

response_json = json.loads(response.text)

if response_json["message"] == "error":

response_json = get_response_json()

return response_json

# 抓取头条热点新闻数据，存入json备用

def data_to_file():

data = get_response_json()["data"]

for i in range(len(data)):

data_dict = data[i]

with open("toutiao.json", "a+") as f:

json.dump(data_dict, f, ensure_ascii=False)

f.write("\n")

df = pd.read_json("toutiao.json", lines=True)

df.to_excel("toutiao.xlsx")

# 抓取热门文章的标签，并存储备用

def get_news_tag():

df = pd.read_json("toutiao.json", lines=True)

tags_pure = []

request_url, headers, proxies = get_request_url_and_headers()

for i in range(len(df)):

news_url = "https://www.toutiao.com" + df["source_url"][i]

# print(news_url)

response = requests.get(news_url, headers=headers, proxies=proxies)

reg = r'"name":".*?"}'

tags = re.findall(reg, response.text)

tags_pure_row = []

if len(tags) <= 0:

tags_pure.append([])

continue

for j in range(len(tags)):

tags_pure_row.append(tags[j].replace('"name":"', '').replace('"}', ''))

tags_pure.append(tags_pure_row)

df["tags"] = tags_pure

df.to_excel("toutiao_with_tags.xlsx")

if __name__ == '__main__':

# print(get_request_url_and_headers())

# print(get_response_html())

# data_to_file()

get_news_tag()

实现展示