第一周第三节练习项目
总结
- 使用BeautifulSoup库可以很方便地处理网页
- 基本套路是用select确定元素位置，get()和get_text()方法提取数据
- 使用User-Agent和Cookie可以”欺骗“服务器
- 爬取网页时要注意通过延时来避免暴露身份
任务
（1）爬取小猪短租网站上一条房源信息
（2）爬取小猪短租网站上300条房源信息
任务1代码
from bs4 import BeautifulSoup
import requests
import time
url='http://bj.xiaozhu.com/fangzi/3686435130.html'
web_data = requests.get(url)
soup = BeautifulSoup(web_data.text,'lxml')
titles = soup.select('div.pho_info > h4 > em')
addresses = soup.select('span.pr5')
prices = soup.select('div.day_l')
room_photos = soup.select('div.pho_show_l > div > div:nth-of-type(2) > img')
lord_photos = soup.select('div.member_pic > a > img')
lord_names = soup.select('a.lorder_name')
lord_genders = soup.select('div.js_box.clearfix > div.w_240 > h6 > span')
for title,address,price,room_photo,lord_photo,lord_name,lord_gender in zip(titles,addresses,prices,room_photos,lord_photos,lord_names,lord_genders):
if lord_gender.get('class')[0] == 'member_girl_ico':
lord_gender = '美女'
else:
lord_gender = '帅哥'
data = {
'title':title.get_text(),
'address':address.get_text().split('\n')[0],
'price':price.get_text()[1:],
'room_photo':room_photo.get('src'),
'lord_photos':lord_photo.get('src'),
'lord_name':lord_name.get_text(),
'lord_gender':lord_gender
}
print(data)
任务1运行结果
1.jpg
任务2代码
from bs4 import BeautifulSoup
import requests
import time
urls = ['http://bj.xiaozhu.com/search-duanzufang-p{}-0/'.format(str(i)) for i in range(1,14,1)]
def get_page(url):
link_list=[]
web_data = requests.get(url)
soup = BeautifulSoup(web_data.text,'lxml')
links = soup.select('ul > li > div.result_btm_con.lodgeunitname')
for link in links:
link_url = link.get('detailurl')
#print (link_url)
link_list.append(link_url)
return(link_list)
def get_info_from_page(url):
web_data = requests.get(url)
soup = BeautifulSoup(web_data.text, 'lxml')
titles = soup.select('div.pho_info > h4 > em')
addresses = soup.select('span.pr5')
prices = soup.select('div.day_l')
room_photos = soup.select('div.pho_show_l > div > div:nth-of-type(2) > img')
lord_photos = soup.select('div.member_pic > a > img')
lord_names = soup.select('a.lorder_name')
lord_genders = soup.select('div.js_box.clearfix > div.w_240 > h6 > span')
for title, address, price, room_photo, lord_photo, lord_name, lord_gender in zip(titles, addresses, prices,
room_photos, lord_photos,
lord_names, lord_genders):
if lord_gender.get('class')[0] == 'member_girl_ico':
lord_gender = '美女'
else:
lord_gender = '帅哥'
data = {
'title': title.get_text(),
'address': address.get_text().split('\n')[0],
'price': price.get_text()[1:],
'room_photo': room_photo.get('src'),
'lord_photos': lord_photo.get('src'),
'lord_name': lord_name.get_text(),
'lord_gender': lord_gender
}
return(data)
links_to_read=[]
room_info=[]
for single_url in urls:
links_to_read = links_to_read + get_page(single_url)
time.sleep(4)
print('共爬取',len(links_to_read),'条房源信息')
for crawl_url in links_to_read:
room_info.append(get_info_from_page(crawl_url))
time.sleep(4)
print(room_info)
任务2运行结果
1.jpg
