场景：由于代码爱好，经常会在github上收集一些有意思的仓库，但是时间一长，会收集很多，也不方便一个一个点击看哪些更新了
需求：抓取仓库更新时间写入excel

#! /usr/bin/python2.7
# -- coding:utf-8 --
import requests
import re
import  xlrd
import  xlwt

# 从一个网页里按正则获取对应的字符串集合
def getIDStringFromWebAtUrlString(urlStringOrHtmlText,regString):
    htmlText = urlStringOrHtmlText
    if urlStringOrHtmlText.startswith('http'):
        htmlText = requests.get(urlStringOrHtmlText).text
    commentsIDStrings = re.findall(regString,htmlText,re.S)
    return commentsIDStrings

# 从一个git仓库获取取最新的更新时间
def getGitRepositoryUpdateTimeTupleWithGitRepositoryUrlStr(gitRepositoryUrlStr):
    dateModifiedRegStr = 'Latest commit(.*?)<a class="commit-tease-sha" href="(.*?)</a>(. *?)<span itemprop="dateModified"><relative-time datetime="(.*?)">'
    dateModifiedTuple =  getIDStringFromWebAtUrlString(gitRepositoryUrlStr, dateModifiedRegStr)
    if len(dateModifiedTuple):
        if len(dateModifiedTuple[0]) >= 4:
            return (gitRepositoryUrlStr, dateModifiedTuple[0][3])

# 从本地的html文件中获取文本
def getHtmlTextFromLocalHtmlFilAtPath(localHtmlFilePath):
    with open(r'%s'%localHtmlFilePath, "r") as f:
        page = f.read()
    return page

# 从本地导出的浏览器书签里获取有关github的所有链接地址
def getGitRepositoryUrlStrsFromLocalBrowserBookmarkAtPath(localBrowserBookmarkFilePath):
    localHtmlText =  getHtmlTextFromLocalHtmlFilAtPath(localBrowserBookmarkFilePath)
    gitRepositoryRegStr = '<DT><A HREF="(.*?)" ADD_DATE="(.*?)" ICON="(.*?)">(.*?)</A>'
    gitRepositoryUrlTupls =  getIDStringFromWebAtUrlString(localHtmlText, gitRepositoryRegStr)
    gitRepositoryUrlStrs = []
    for gitRepositoryUrlTupl in gitRepositoryUrlTupls:
        if gitRepositoryUrlTupl[0].startswith('https://github.com/'):
            gitRepositoryUrlStrs.append(gitRepositoryUrlTupl[0])
    return gitRepositoryUrlStrs



def getGitRepositoryUpdateTimeTuplesFromLocalBrowserBookmarkAtPath(localBrowserBookmarkFilePath):
    getGitRepositoryUrlStrs =  getGitRepositoryUrlStrsFromLocalBrowserBookmarkAtPath(localBrowserBookmarkFilePath)
    gitRepositoryUpdateTimeTuples = []
    for getGitRepositoryUrlStr in getGitRepositoryUrlStrs:
        gitRepositoryUpdateTimeTuple = getGitRepositoryUpdateTimeTupleWithGitRepositoryUrlStr(getGitRepositoryUrlStr)
        if gitRepositoryUpdateTimeTuple is None:
            pass
        else:
            gitRepositoryUpdateTimeTuples.append(gitRepositoryUpdateTimeTuple)
    return gitRepositoryUpdateTimeTuples

def getGitRepositoryUpdateTimeFromfilePathAndSaveName(localBrowserBookmarkFilePath, saveExcelName):
    book = xlwt.Workbook()
    sheet1 = book.add_sheet('hello')
    gitRepositoryUpdateTimeTuples = getGitRepositoryUpdateTimeTuplesFromLocalBrowserBookmarkAtPath(localBrowserBookmarkFilePath)
    for index, gitRepositoryUpdateTimeTuple in enumerate(gitRepositoryUpdateTimeTuples):
        sheet1.write(index, 0, gitRepositoryUpdateTimeTuple[0])
        sheet1.write(index, 1, gitRepositoryUpdateTimeTuple[1])
    rightSaveExcelName = saveExcelName
    if saveExcelName.endswith('.xls'):
        pass
    else:
        rightSaveExcelName = saveExcelName + '.xls'
    book.save(rightSaveExcelName)

# 开始调用吧
getGitRepositoryUpdateTimeFromfilePathAndSaveName('/Users/dfpo/Desktop/bookmarks_2017_9_10.html', 'githubRepUpdate.xls')

成果展示

image.png