原理：

• 通过改变训练样本的权重，学习多个分类器，并将这些分类器进行线性组合，提高分类的性能；
• bagging是通过随机抽样的替换方式，得到与原数据集规模一样的数据；
• boosting在bagging的思路上更进一步，在数据集上顺序应用了多个不同的分类器。

优点：

• 泛化错误率低，易编码，可以用在大部分分类器上，无参数调整。

缺点：

• 对离群点敏感

适用数据类型：

• 数值型和标称型数据

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
def loadSimpData():
    dataMat = np.mat([[1.,2.1],
               [2.,1.1],
               [1.3,1.],
               [1.,1.],
               [2.,1.]])
    classLabels = [1.0,1.0,-1.0,-1.0,1.0]
    return dataMat,classLabels
dataMat,classLabels=loadSimpData()

#可视化
x = pd.DataFrame(dataMat)
y= pd.Series(classLabels)
pos = x[y==1].copy()
neg = x[y==-1].copy()
plt.figure()
plt.scatter(pos.loc[:,0],pos.loc[:,1],c='b',label='positive',marker='s')
plt.scatter(neg.loc[:,0],neg.loc[:,1],c='r',label='negative')
plt.show()

output_1_0.png

分类规则

def stumpClassify(dataMatrix,dimen,threshVal,threshIneq):
    retArray = np.ones((np.shape(dataMatrix)[0],1))
    if threshIneq == 'lt':
        retArray[dataMatrix[:,dimen] <= threshVal] =-1.0  #大小判断进行分类
    else: #threshIneq == 'gt'
        retArray[dataMatrix[:,dimen] > threshVal] =-1.0
    return retArray

遍历所有特征找到最小误差对应的特征、阈值和分类规则

def buildStump(dataArr,classLabels,D):
    dataMatrix = np.mat(dataArr);labelMat = np.mat(classLabels).T
    m,n=np.shape(dataMatrix)
    numSteps = 10.0; bestStump = {}; bestClassEst = np.mat(np.zeros((m,1)))
    minError=np.inf
    for i in range(n):
        rangeMin = dataMatrix[:,i].min() #取第i个特征的最小值
        rangeMax = dataMatrix[:,i].max() #取第i个特征的最大值
        stepSize = (rangeMax - rangeMin)/numSteps #设置步长
        for j in range(-1,int(numSteps)+1): #j：-1到11，用于遍历不同的阈值
            for inequal in ['lt','gt']: #lt:less than; gt: great than
                threshVal = rangeMin + float(j)*stepSize #设置阈值，按一个步长一个步长往上增加
                predictedVals = stumpClassify(dataMatrix,i,threshVal,inequal) #分类结果
                errArr = np.mat(np.ones((m,1))) #初始化误差矩阵
                errArr[predictedVals == labelMat] = 0 #正确分类的误差为0
                weightedError = D.T*errArr #计算分类误差率，参考李航P139
#                 print("split: dim %d, thresh %.2f, thresh ineqal: %s, the weighted error is %.3f" %(i,threshVal,inequal,weightedError))
                if weightedError < minError:
                    minError = weightedError #找到最小误差
                    bestClassEst = predictedVals.copy() #最小误差对应的分类结果
                    bestStump['dim'] = i #最小误差对应的特征
                    bestStump['thresh'] = threshVal #最小误差对应的阈值
                    bestStump['ineq'] = inequal #最小误差对应的规则
    return bestStump,minError,bestClassEst

D = np.mat(np.ones((5,1))/5)
print(D)
buildStump(dataMat,classLabels,D)

[[0.2]
 [0.2]
 [0.2]
 [0.2]
 [0.2]]





({'dim': 0, 'thresh': 1.3, 'ineq': 'lt'}, matrix([[0.2]]), array([[-1.],
        [ 1.],
        [-1.],
        [-1.],
        [ 1.]]))

完整版AdaBoost算法实现

from math import *
def adaBoostTrainDS(dataArr,classLabels,numIt=40):
    weakClassArr = []
    m = np.shape(dataArr)[0]
    D = np.mat(np.ones((m,1))/m)
    aggClassEst = np.mat(np.zeros((m,1)))
    for i in range(numIt):
        #找到最佳单层决策树
        bestStump,error,classEst = buildStump(dataArr,classLabels,D)#classEst:array
#         print('D:\n',D.T)
        
        #更新alpha，weight
        alpha = float(0.5*log((1.0-error)/max(error,1e-16)))
        bestStump['alpha'] = alpha 
        weakClassArr.append(bestStump)
#         print('classEst:\n',classEst.T)
        expon = np.multiply(-1*alpha*np.mat(classLabels).T,classEst) #matrix
        D = np.multiply(D,np.exp(expon))
        D = D/D.sum()
        
        #记录每个样布的评估结果，可以看到每个样本的评估都在往一个正确的方向变化
        aggClassEst += alpha*classEst
#         print('aggClassEst:\n',aggClassEst)
        
        #统计错误率
        aggErrors = np.multiply(np.sign(aggClassEst) != np.mat(classLabels).T,np.ones((m,1))) #统计错误个数
        errorRate = aggErrors.sum() / m#计算错误率
        print('total error:\n',errorRate)
        
        #若错误率为0，则停止迭代
        if errorRate == 0.0: break
    return weakClassArr,aggClassEst

classifierArray,aggClassEst = adaBoostTrainDS(dataMat,classLabels,9)
classifierArray

total error:
 0.2
total error:
 0.2
total error:
 0.0





[{'dim': 0, 'thresh': 1.3, 'ineq': 'lt', 'alpha': 0.6931471805599453},
 {'dim': 1, 'thresh': 1.0, 'ineq': 'lt', 'alpha': 0.9729550745276565},
 {'dim': 0, 'thresh': 0.9, 'ineq': 'lt', 'alpha': 0.8958797346140273}]

测试算法

def adaClassify(datToclass,classifierArr):
    dataMatrix = np.mat(datToclass)
    m = np.shape(dataMatrix)[0]
    aggClassEst = np.mat(np.zeros((m,1)))
    for i in range(len(classifierArray)): #遍历全部弱分类器
        classEst = stumpClassify(dataMatrix,classifierArr[i]['dim'],classifierArray[i]['thresh'],classifierArray[i]['ineq']) #分类结果：1，-1
        aggClassEst += classifierArray[i]['alpha']*classEst
#         print(aggClassEst)
    res =np.sign(aggClassEst)
    return res

#测试
adaClassify([0,0],classifierArray)

matrix([[-1.]])

在马疝病数据集应用AdaBoost分类器

def loadDataSet(fileName):
    fr = open(fileName)
    numFeature = len(fr.readline().strip().split('\t'))
    dataMat =[];labelMat =[]
    for line in fr.readlines():
        lineList =[]
        curLine = line.strip().split('\t')
        for i in range(numFeature-1):
            lineList.append(float(curLine[i]))
        dataMat.append(lineList)
        labelMat.append(float(curLine[-1]))
    return dataMat,labelMat

traindataMat,trainlabelMat = loadDataSet('../../Reference Code/Ch07/horseColicTraining2.txt')
testdataMat,testlabelMat = loadDataSet('../../Reference Code/Ch07/horseColicTest2.txt')

classifierArray,aggClassEst = adaBoostTrainDS(traindataMat,trainlabelMat,10)
classifierArray

total error:
 0.28523489932885904
total error:
 0.28523489932885904
total error:
 0.2483221476510067
total error:
 0.2483221476510067
total error:
 0.2483221476510067
total error:
 0.24161073825503357
total error:
 0.24161073825503357
total error:
 0.2214765100671141
total error:
 0.2483221476510067
total error:
 0.2214765100671141





[{'dim': 9, 'thresh': 3.0, 'ineq': 'gt', 'alpha': 0.4593204546095544},
 {'dim': 17, 'thresh': 52.5, 'ineq': 'gt', 'alpha': 0.31654488263333286},
 {'dim': 3,
  'thresh': 55.199999999999996,
  'ineq': 'gt',
  'alpha': 0.28402835050611847},
 {'dim': 18,
  'thresh': 62.300000000000004,
  'ineq': 'lt',
  'alpha': 0.23222873860913737},
 {'dim': 10, 'thresh': 0.0, 'ineq': 'lt', 'alpha': 0.19836267426245105},
 {'dim': 5, 'thresh': 2.0, 'ineq': 'gt', 'alpha': 0.18642416210017293},
 {'dim': 12, 'thresh': 1.2, 'ineq': 'lt', 'alpha': 0.1496988869138094},
 {'dim': 7, 'thresh': 1.2, 'ineq': 'gt', 'alpha': 0.15848275395378547},
 {'dim': 5, 'thresh': 0.0, 'ineq': 'lt', 'alpha': 0.1370746524177519},
 {'dim': 0, 'thresh': 1.0, 'ineq': 'lt', 'alpha': 0.12365372615766472}]

prediction = adaClassify(testdataMat,classifierArray)
errArr = np.ones((len(prediction),1))
errNum = errArr[prediction!=np.mat(testlabelMat).T].sum()
errRate= errNum/len(prediction)
print('错误个数：%d'%errNum)
print('错误率：%.2f'%errRate)

错误个数：15
错误率：0.23

ROC曲线

def plotROC(predStrengths,classLabels):
    import matplotlib.pyplot as plt
    cur = (0.,0.)
    ySum =0.
    numPosClass = sum(np.array(classLabels)==1.0) #统计正例的数目
    yStep = 1/ float(numPosClass) #正阳率的步长
    xStep = 1/float(len(classLabels)-numPosClass) #假阳率的步长
    sortedIndicies = predStrengths.argsort() #从小到大排序，返回index
    fig = plt.figure()
#     fig.clf()
    ax = plt.subplot(111)
    #[::-1],反转，从大到小排序，即所有样例判定为反例
    for index in sortedIndicies.tolist()[0][::-1]: #tolist(),matrix 变成list，才能遍历里面的元素
        if classLabels[index] == 1.0: #若当前为正例，则正阳率增加一个步长，假阳率不变
            delX=0;delY=yStep
        else:
            delX = xStep;delY=0
            ySum+=cur[1]
        ax.plot([cur[0] , cur[0]+delX],[cur[1],cur[1]+delY],c='b')
        cur = (cur[0] + delX,cur[1] + delY) #当前样例的坐标
    ax.plot([0,1],[0,1],'b--')
    plt.xlabel('False Positive Rate')
    plt.ylabel('True Positive Rate')
    ax.axis([0,1,0,1])
    plt.show()
    print('the Area Under the Curve is :',ySum*xStep)

traindataMat,trainlabelMat = loadDataSet('../../Reference Code/Ch07/horseColicTraining2.txt')
classifierArray,aggClassEst = adaBoostTrainDS(traindataMat,trainlabelMat,10)

total error:
 0.28523489932885904
total error:
 0.28523489932885904
total error:
 0.2483221476510067
total error:
 0.2483221476510067
total error:
 0.2483221476510067
total error:
 0.24161073825503357
total error:
 0.24161073825503357
total error:
 0.2214765100671141
total error:
 0.2483221476510067
total error:
 0.2214765100671141

plotROC(aggClassEst.T,trainlabelMat) #aggClassEst.T变成一行

output_21_0.png

the Area Under the Curve is : 0.8538389513108627