仔细研究网易云音乐排行榜html码源，发现榜单是iframe动态加载的，因此scrapy结合selenium来爬取
- 在middlewares.py文件中建立中间件，结合selenium
from selenium import webdriver
import selenium.webdriver.support.ui as ui
from scrapy.http import HtmlResponse
#采用中间件结合selenium
class JavaScriptMiddleware(object):
def process_request(self, request, spider):
if spider.name == "wymusic":
print "PhantomJS is starting..."
driver = webdriver.PhantomJS() #指定使用的浏览器
driver.get(request.url)
driver.switch_to.frame('g_iframe') #移动到 iframe
wait = ui.WebDriverWait(driver, 15)
if wait.until(lambda driver: driver.find_element_by_id('toplist')):
body = driver.page_source
return HtmlResponse(driver.current_url, body=body, encoding='utf-8', request=request)
else:
return
- 在settings.py中添加中间件
DOWNLOADER_MIDDLEWARES = {
#'scrapySpider.middlewares.MyCustomDownloaderMiddleware': 543,
'wymusicSpider.middlewares.JavaScriptMiddleware': 543,
}
- 在items.py中定义要爬取的字段
class WymusicItem(scrapy.Item):
rank = scrapy.Field() # 排名
title = scrapy.Field() # 歌名
duration = scrapy.Field() # 时长
author = scrapy.Field() # 作者
- 编写爬虫
import scrapy
from scrapy.http.request import Request
from wymusicSpider.items import WymusicItem
import sys
import lxml
reload(sys)
sys.setdefaultencoding('utf-8')
#html DOM tree to lxml 格式
def parse_from_unicode(unicode_str):
utf8_parser = lxml.etree.HTMLParser(encoding='utf-8')
s = unicode_str.encode('utf-8')
return lxml.etree.fromstring(s, parser=utf8_parser)
class WymusicSpider(scrapy.Spider):
name = 'wymusic'
allowed_domains = ['music.163.com']
start_urls = ['http://music.163.com/#/discover/toplist']
def parse(self, response):
items = []
dom = parse_from_unicode(response.body)
lists = dom.xpath('//tbody/tr')
for row in lists:
td = row.xpath("td")
rank = td[0].xpath("div/span[@class='num']/text()")[0]
title = td[1].xpath("div/div/div/span/a/b//@title")[0]
duration = td[2].xpath("span/text()")[0]
author = td[3].xpath("div//@title")[0]
item = WymusicItem()
item['rank'] = rank
item['title'] = title
item['duration'] = duration
item['author'] = author
if item not in items:
items.append(item)
return items
