一、背景与目标

• 爬国外一个竞猜网站，世界杯的 的竞猜数据。

二、实现

• 下载chromedriver

1. 翻墙工具是扩展程序 Ghelper

• 因为翻墙工具 是扩容程序
  • 扩展程序要输出账号密码
    • user-data-dir 参数制定用现有的账号密码

1.1 google 浏览器

• 碰到的问题 无界面浏览模式与自定义插件加载问题
  • google-chrome-head 不支持扩展。

from selenium import webdriver
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
import time

# 创建浏览器对象

# driver = webdriver.Chrome()

from selenium import webdriver
import time

# 启动浏览器
# url = 'https://www.toutiao.com'

url = 'https://sports.ladbrokes.com/competitions/football/english/premier-league'

chrome_options = webdriver.ChromeOptions()

# 设置好应用扩展
extension_path = 'C:/Users/lizhi/Downloads/Ghelper-v2.6.2/Ghelper-v2.6.2.crx'
chrome_options.add_extension(extension_path)
#
chrome_options.add_argument('--headless')
# chrome_options.add_argument('--no-sandbox')
# chrome_options.add_argument('--disable-dev-shm-usage')

chrome_options.add_argument(r'user-data-dir=C:\Users\lizhi\AppData\Local\Google\Chrome\User Data')
# chrome_options.add_experimental_option('useAutomationExtension', False)
# chrome_options.add_argument(r'user-data-dir=C:\Users\lenovo\AppData\Local\Google\Chrome\User Data')



driver = webdriver.Chrome(r'C:\Program Files\Google\Chrome\Application\chromedriver.exe',chrome_options=chrome_options)

# driver = webdriver.Chrome("chromedriver.exe")



driver.get(url)
driver.implicitly_wait(10)
driver.maximize_window()  # 最大化窗口
driver.implicitly_wait(10)
driver.find_element_by_link_text('热点').click()  # 点击热点
driver.implicitly_wait(10)

title_list = []


def scoll():  # 滚动页面
    # driver.execute_script("window.scrollTo(0,1000);")
    # time.sleep(1)
    name = driver.find_elements_by_class_name("container-inner-content")
    print(name)


# 运行scoll函数
scoll()
print(title_list)  # 输出结果

1.2 google 不行我换火狐浏览器

• 不行，我翻墙工具是 google插件

1.3 换 Microsoft Edge 浏览器

• Microsoft Edge 能直接使用 google 的扩展程序，翻墙没问题
• 无界面浏览模式与自定义插件加载问题 ie 也有

# 创建浏览器对象

# driver = webdriver.Chrome()

from msedge.selenium_tools.service import Service
from selenium import webdriver
# from msedge.selenium_tools import Edge, EdgeOptions

from selenium.webdriver.edge.options import Options

# 启动浏览器
# url = 'https://www.toutiao.com'

url = 'https://sports.ladbrokes.com/competitions/football/english/premier-league'


from selenium.webdriver.edge.options import Options

options = Options()
 # options.add_argument("headless")
options.add_argument('headless')
options.add_argument('no-sandbox')
options.add_argument('disable-dev-shm-usage')
# options = EdgeOptions()
# options.use_chromium = True
# options.add_argument("headless")

# 设置好应用扩展
extension_path = 'C:/Users/lizhi/Downloads/Ghelper-v2.6.2/Ghelper-v2.6.2.crx'
options.add_extension(extension_path)
options.add_argument(r'user-data-dir=C:\Users\lizhi\AppData\Local\Microsoft\Edge\User Data')


# from selenium.webdriver.edge.service import Service
service = Service(r'C:\Users\lizhi\Downloads\edgedriver_win64\msedgedriver.exe', verbose=True)

driver = webdriver.Edge(service=service, options=options)


# driver = webdriver.Chrome("chromedriver.exe")



driver.get(url)
driver.implicitly_wait(10)
driver.maximize_window()  # 最大化窗口
driver.implicitly_wait(10)
driver.find_element_by_link_text('热点').click()  # 点击热点
driver.implicitly_wait(10)

title_list = []


def scoll():  # 滚动页面
    # driver.execute_script("window.scrollTo(0,1000);")
    # time.sleep(1)
    name = driver.find_elements_by_class_name("container-inner-content")
    print(name)


# 运行scoll函数
scoll()
print(title_list)  # 输出结果

2. 翻墙工具 用VPN程序

• 爬取网站
  • https://sports.ladbrokes.com/
• 数据页
  • image.png

2.1 代码

from selenium import webdriver
from selenium.webdriver.chrome.service import Service
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions
import time

# 创建浏览器对象

from selenium import webdriver
import time

# 启动浏览器
# url = 'https://www.toutiao.com'

url = 'https://sports.ladbrokes.com/competitions/football/english/premier-league'

chrome_options = webdriver.ChromeOptions()

# 设置好应用扩展
# extension_path = 'C:/Users/lizhi/Downloads/Ghelper-v2.6.2/Ghelper-v2.6.2.crx'
# chrome_options.add_extension(extension_path)
#
# chrome_options.add_argument('--headless')
# chrome_options.add_argument('--no-sandbox')
# chrome_options.add_argument('--disable-dev-shm-usage')
#
# chrome_options.add_argument(r'user-data-dir=C:\Users\lizhi\AppData\Local\Google\Chrome\User Data')
# chrome_options.add_experimental_option('useAutomationExtension', False)
# chrome_options.add_argument(r'user-data-dir=C:\Users\lenovo\AppData\Local\Google\Chrome\User Data')


service = Service(r'C:\Users\lizhi\Downloads\chromedriver_win32 (1)\chromedriver.exe')

driver = webdriver.Chrome(service=service)

# driver = webdriver.Chrome("chromedriver.exe")



driver.get(url)
driver.implicitly_wait(10)
driver.maximize_window()  # 最大化窗口
driver.implicitly_wait(10)
# # driver.find_element_by_link_text('热点').click()  # 点击热点
# driver.implicitly_wait(10)

title_list = []


def scoll():  # 滚动页面 class="sport-card"
    driver.execute_script("window.scrollTo(0,1000);")
    time.sleep(1)
    from selenium.webdriver.common.by import By
    names = driver.find_elements(by=By.CLASS_NAME, value="container-inner-content")
    for name in names:
        month = name.find_element(by=By.CLASS_NAME, value="odds-left").find_element(by=By.TAG_NAME,value="span").get_attribute("innerText")
        elements = name.find_elements(by=By.TAG_NAME, value="odds-card-sport")
        print(f'爬取日期={month}')
        for element in elements:
            header_element = element.find_element(by=By.CLASS_NAME,value="sport-card-header")
            data_time = header_element.find_element(by=By.CLASS_NAME,value="sport-card-left").find_element(by=By.TAG_NAME,value="span").get_attribute("innerText")
            data_score = header_element.find_element(by=By.CLASS_NAME, value="sport-card-right").find_element(
                by=By.TAG_NAME, value="a").get_attribute("innerHTML")

            team_elements = element.find_element(by=By.CLASS_NAME,value="sport-card-content").find_element(by=By.CLASS_NAME, value="sport-card-left").find_elements(
                by=By.TAG_NAME, value="a")

            team1 = team_elements[0].get_attribute("innerHTML")
            team2 = team_elements[0].get_attribute("innerHTML")
            print(f'比赛日期={data_time},data_score={data_score},teamA={team1},teamB={team2}')




# 运行scoll函数
scoll()

2.2 输出

image.png

爬取日期=5 Nov
比赛日期=23:00 05 Nov,data_score=153 MORE,teamA=Leeds,teamB=Leeds
比赛日期=23:00 05 Nov,data_score=153 MORE,teamA=Man City,teamB=Man City
比赛日期=23:00 05 Nov,data_score=153 MORE,teamA=Nottm Forest,teamB=Nottm Forest
比赛日期=23:00 05 Nov,data_score=153 MORE,teamA=Wolves,teamB=Wolves
爬取日期=6 Nov
比赛日期=01:30 06 Nov,data_score=153 MORE,teamA=Everton,teamB=Everton
比赛日期=20:00 06 Nov,data_score=153 MORE,teamA=Chelsea,teamB=Chelsea
比赛日期=22:00 06 Nov,data_score=153 MORE,teamA=Aston Villa,teamB=Aston Villa
比赛日期=22:00 06 Nov,data_score=153 MORE,teamA=Southampton,teamB=Southampton
比赛日期=22:00 06 Nov,data_score=153 MORE,teamA=West Ham,teamB=West Ham
爬取日期=7 Nov
比赛日期=00:30 07 Nov,data_score=153 MORE,teamA=Tottenham,teamB=Tottenham
爬取日期=12 Nov
比赛日期=20:30 12 Nov,data_score=153 MORE,teamA=Man City,teamB=Man City
比赛日期=23:00 12 Nov,data_score=153 MORE,teamA=Bournemouth,teamB=Bournemouth
比赛日期=23:00 12 Nov,data_score=153 MORE,teamA=Liverpool,teamB=Liverpool
比赛日期=23:00 12 Nov,data_score=152 MORE,teamA=Nottm Forest,teamB=Nottm Forest
比赛日期=23:00 12 Nov,data_score=153 MORE,teamA=Tottenham,teamB=Tottenham
比赛日期=23:00 12 Nov,data_score=152 MORE,teamA=West Ham,teamB=West Ham
爬取日期=13 Nov
比赛日期=01:30 13 Nov,data_score=153 MORE,teamA=Newcastle,teamB=Newcastle
比赛日期=03:45 13 Nov,data_score=153 MORE,teamA=Wolves,teamB=Wolves
比赛日期=22:00 13 Nov,data_score=153 MORE,teamA=Brighton,teamB=Brighton
爬取日期=14 Nov
比赛日期=00:30 14 Nov,data_score=153 MORE,teamA=Fulham,teamB=Fulham

• 界面上能看到数据，就能爬到，所见即可得。