1. 引言
程序会因为各种各样的原因而中断执行, 所以要想办法主程序在上次中断的地方继续执行
2. 分析
- 随时存储程序执行的状态值, 例如程序执行到的位置
- 程序再次执行时先获取上次中断前保存的位置
3. 实现部分
代码
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
__author__ = 'jhw'
from bs4 import BeautifulSoup
from pymongo import MongoClient
import requests
# 连接mongo数据库
client = MongoClient('10.66.17.17', 27017)
database = client['58tongcheng']
# 存放url信息的collection
url_info = database['shoujihao_url']
# 存放item信息的collection
item_info = database['shoujihao_item']
# 存放已打开的url
url_exists = database['shoujihao_url_exists']
# 获取快打开的url列表, 以便在get_item_info函数中作对比
exists_list = [i['url'] for i in url_exists.find()]
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.106 Safari/537.36',
}
# 定义获取手机号信息的函数
def get_url_info(url):
data = requests.get(url, headers=headers)
soup = BeautifulSoup(data.text, 'lxml')
# 判断是否到了列表尾部
if len(soup.select('.boxlist a.t')) == 0:
print('*'*20, 'End of the pages...', '*'*20)
print(url)
return 1 else: titles = soup.select('.boxlist a.t > strong') urls = soup.select('.boxlist a.t') for title, url in zip(titles, urls): # 过虑推广信息 if 'jump' in url.get('href'): print('Advertising info, Pass...') else: data = { 'title': title.get_text(), 'url': url.get('href').split('?')[0], } print(data) # 将url信息插入mongodb中 url_info.insert_one(data)
def get_item_info(url):
# 如果即将要打开的url存在于已打开的url列表中则退出本次循环
if url in exists_list:
print('*'*20, 'The url has been opened before!!!', '*'*20)
else:
data = requests.get(url, headers=headers)
soup = BeautifulSoup(data.text, 'lxml')
titles = soup.select('.mainTitle h1')
updates = soup.select('li.time')
prices = soup.select('.price.c_f50')
data = {
# 去掉多余的字符
'title': titles[0].get_text().replace('\n', '').replace('\t', '').replace(' ', ''),
'price': prices[0].get_text().replace('\n', '').replace('\t', '').replace(' ', ''),
'update': updates[0].get_text().replace('\n', '').replace('\t', '').replace(' ', ''),
}
print(data)
# 将item信息插入mongodb中
item_info.insert_one(data)
url_exists.insert_one({'url': url})
# 定义包含所有列表页的列表
url_list = ['http://bj.58.com/shoujihao/pn{}/'.format(i) for i in range(1, 200)]
# 逐一抓取列表页中的手机号信息
# for url in url_list:
# # 至列表尾部了就退出
# if get_url_info(url):
# break
# 从mongodb中取出url, 获取其详细信息
for url in url_info.find():
get_item_info(url['url'])
结果
******************** The url has been opened before!!! ********************
******************** The url has been opened before!!! ********************
******************** The url has been opened before!!! ********************
******************** The url has been opened before!!! ********************
******************** The url has been opened before!!! ********************
{'update': '2016-07-06', 'price': '3325元', 'title': '13341143344联通北京靓号'}
{'update': '2016-07-06', 'price': '5650元', 'title': '15001162111移动北京壹手靓号'}
{'update': '2016-07-06', 'price': '10640元', 'title': '18511111990联通北京顶级靓号'}
{'update': '2016-07-06', 'price': '5645元', 'title': '18511188558移动北京壹手靓号'}
{'update': '2016-07-06', 'price': '10640元', 'title': '18601111100联通北京顶级靓号'}
{'update': '2016-07-06', 'price': '5686元', 'title': '13810376555移动高价回收靓号'}
4. 总结
- 程序执行时随时保存执行状态, 如果别的地方有需要用到就很方便了