%matplotlib inline
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import matplotlib.gridspec as gridspec
pd.set_option("display.max_columns",101)
pd.set_option('display.float_format', lambda x: '%.5f' % x) #为了直观的显示数字，不采用科学计数法
pd.options.display.max_rows = 15 #最多显示15行
import warnings
warnings.filterwarnings('ignore') #为了整洁，去除弹出的warnings
import pandas as pd
df=pd.read_csv( 'cs-training.csv')
df = df.drop(df.columns[0],axis=1)
df=df[df.age>=18]
在债务违约预测之一：数据探索中，按各个属性对借贷者分组，再分析其违约率。现在换一个角度，分为违约者和未违约两类，再查看两组人群中各个属性的分布。
features=df.columns[1:]
features
Index(['RevolvingUtilizationOfUnsecuredLines', 'age',
'NumberOfTime30-59DaysPastDueNotWorse', 'DebtRatio', 'MonthlyIncome',
'NumberOfOpenCreditLinesAndLoans', 'NumberOfTimes90DaysLate',
'NumberRealEstateLoansOrLines', 'NumberOfTime60-89DaysPastDueNotWorse',
'NumberOfDependents'],
dtype='object')
plt.figure(figsize=(12,28*4))
gs = gridspec.GridSpec(28, 1)
#针对违约者和未违约者的每个属性，绘制直方图
for i, cn in enumerate(features):
ax = plt.subplot(gs[i])
sns.distplot(df[cn][df.SeriousDlqin2yrs == 1], bins=50,color='red')
sns.distplot(df[cn][df.SeriousDlqin2yrs == 0], bins=50,color='blue')
ax.set_xlabel('')
ax.set_title('histogram of feature: ' + str(cn))
plt.show()
出现 'max must be larger than min in range parameter.'是因为有的列存在空值。
df.isnull().sum()
MonthlyIncome为空的记录较多，为了保持数据的完整，没有删掉，用平均值填充
df['MonthlyIncome'].fillna(df['MonthlyIncome'].mean(), inplace=True)
df['NumberOfDependents'].fillna(df['NumberOfDependents'].mode(), inplace=True)
#NumberOfDependents字段，用众数df['NumberOfDependents'].mode()来填充
df.isnull().sum() #空值还是存在，为什么呢
SeriousDlqin2yrs 0
RevolvingUtilizationOfUnsecuredLines 0
age 0
NumberOfTime30-59DaysPastDueNotWorse 0
DebtRatio 0
MonthlyIncome 0
NumberOfOpenCreditLinesAndLoans 0
NumberOfTimes90DaysLate 0
NumberRealEstateLoansOrLines 0
NumberOfTime60-89DaysPastDueNotWorse 0
NumberOfDependents 3924
dtype: int64
type(df['NumberOfDependents'].mode())
pandas.core.series.Series
#mode()返回的是一个Series，而不是单一的值,要取其中的元素来填充
df['NumberOfDependents'].fillna(df['NumberOfDependents'].mode()[0], inplace=True)#填补成功
sns.distplot(df['RevolvingUtilizationOfUnsecuredLines'][(df.SeriousDlqin2yrs == 1) & (df.RevolvingUtilizationOfUnsecuredLines)], bins=20,color='red')
sns.distplot(df['RevolvingUtilizationOfUnsecuredLines'][(df.SeriousDlqin2yrs == 0) & (df.RevolvingUtilizationOfUnsecuredLines)], bins=20,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0x10229a58>
output_15_1.png
图形缩成小小的一条，因为取值范围是0到50000多，x轴的范围太大了，而大部分值都在0附近，所以无法清晰显示。
df['RevolvingUtilizationOfUnsecuredLines'].describe() #看该属性的数值分布
count 149999.00000
mean 6.04847
std 249.75620
min 0.00000
25% 0.02987
50% 0.15418
75% 0.55904
max 50708.00000
Name: RevolvingUtilizationOfUnsecuredLines, dtype: float64
df[['RevolvingUtilizationOfUnsecuredLines']].boxplot(sym='r*') #用箱型图查看异常值
<matplotlib.axes._subplots.AxesSubplot at 0x100f1828>
output_18_1.png
p=df[['RevolvingUtilizationOfUnsecuredLines']].boxplot(return_type='dict')
#return_type='dict'时，会返回数据集的异常值
outliers=p['fliers'][0].get_xydata()#get_xydata()把异常值返回到一个二维数组中
outliers.shape
(763, 2)
outliers[:,1:].min() #看看最小的异常值是多少
1.3534146969999998
sns.distplot(df['RevolvingUtilizationOfUnsecuredLines'][(df.SeriousDlqin2yrs == 1) & (df.RevolvingUtilizationOfUnsecuredLines<1.4)], bins=20,color='red')
sns.distplot(df['RevolvingUtilizationOfUnsecuredLines'][(df.SeriousDlqin2yrs == 0) & (df.RevolvingUtilizationOfUnsecuredLines<1.4)], bins=20,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0x1027f5f8>
output_21_1.png
未违约人群，RevolvingUtilizationOfUnsecuredLines属性的最高频率在0附近；违约人群，RevolvingUtilizationOfUnsecuredLines的最高频率在1附近。
#计算每个属性的异常值数量和最小的异常值
col_min={}
for feature in features:
p=df[[feature]].boxplot(return_type='dict')
outliers=p['fliers'][0].get_xydata()
pmin=outliers[:,1:].min()
col_min[feature]=[outliers.shape[0],pmin]
output_23_0.png
col_min
{'DebtRatio': [31311, 1.9080459769999998],
'MonthlyIncome': [9149, 12646.0],
'NumberOfDependents': [13336, 3.0],
'NumberOfOpenCreditLinesAndLoans': [3980, 21.0],
'NumberOfTime30-59DaysPastDueNotWorse': [23981, 1.0],
'NumberOfTime60-89DaysPastDueNotWorse': [7604, 1.0],
'NumberOfTimes90DaysLate': [8338, 1.0],
'NumberRealEstateLoansOrLines': [793, 6.0],
'RevolvingUtilizationOfUnsecuredLines': [763, 1.3534146969999998],
'age': [45, 97.0]}
#结合异常值和该属性上的数值分布，选定取值范围作图。因为每个属性的选取范围和bins不同，所以不进行统一绘图，
而是一个一个绘制。
sns.distplot(df['DebtRatio'][(df.SeriousDlqin2yrs == 1) & (df.DebtRatio<5)], bins=20,color='red')
sns.distplot(df['DebtRatio'][(df.SeriousDlqin2yrs == 0) & (df.DebtRatio<5)], bins=20,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0xa7cfef0>
output_25_1.png
两组人群在DebtRatio属性上的分布相似，最高频率在0附近，后逐渐降低
sns.distplot(df['NumberOfOpenCreditLinesAndLoans'][(df.SeriousDlqin2yrs == 1) & (df.NumberOfOpenCreditLinesAndLoans<30)], bins=30,color='red')
sns.distplot(df['NumberOfOpenCreditLinesAndLoans'][(df.SeriousDlqin2yrs == 0) & (df.NumberOfOpenCreditLinesAndLoans<30)], bins=30,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0x11eca3c8>
output_27_1.png
在NumberOfOpenCreditLinesAndLoans属性上，两组人群分布相似，最高频率都是5-8之间
sns.distplot(df['age'][df.SeriousDlqin2yrs == 1] ,bins=50,color='red')
sns.distplot(df['age'][df.SeriousDlqin2yrs == 0], bins=50,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0x137bbfd0>
output_29_1.png
sns.distplot(df['NumberOfTime30-59DaysPastDueNotWorse'][(df.SeriousDlqin2yrs == 1) & (df['NumberOfTime30-59DaysPastDueNotWorse']<10)], bins=10,color='red')
sns.distplot(df['NumberOfTime30-59DaysPastDueNotWorse'][(df.SeriousDlqin2yrs == 0) & (df['NumberOfTime30-59DaysPastDueNotWorse']<10)], bins=10,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0xbb45908>
output_30_1.png
sns.distplot(df['NumberOfTime60-89DaysPastDueNotWorse'][(df.SeriousDlqin2yrs == 1) & (df['NumberOfTime60-89DaysPastDueNotWorse']<10)], bins=10,color='red')
sns.distplot(df['NumberOfTime60-89DaysPastDueNotWorse'][(df.SeriousDlqin2yrs == 0) & (df['NumberOfTime60-89DaysPastDueNotWorse']<10)], bins=10,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0xbf33940>
output_31_1.png
sns.distplot(df['NumberOfTimes90DaysLate'][(df.SeriousDlqin2yrs == 1) & (df.NumberOfTimes90DaysLate<10)], bins=10,color='red')
sns.distplot(df['NumberOfTimes90DaysLate'][(df.SeriousDlqin2yrs == 0) & (df.NumberOfTimes90DaysLate<10)], bins=10,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0xa5992b0>
output_32_1.png
sns.distplot(df['NumberRealEstateLoansOrLines'][(df.SeriousDlqin2yrs == 1) & (df.NumberRealEstateLoansOrLines<10)], bins=10,color='red')
sns.distplot(df['NumberRealEstateLoansOrLines'][(df.SeriousDlqin2yrs == 0) & (df.NumberRealEstateLoansOrLines<10)], bins=10,color='blue')
<matplotlib.axes._subplots.AxesSubplot at 0xa7aa2e8>
output_33_1.png
其余几个属性上，两类人群的分布都是相近的，不再赘述。和本文采用的是不同分析方法，
前者按各个属性对借贷者分组，查看不同类别在每一组的分布。本文是先进行分类，再查看两个类别中各个属性的分布。
。第一种方法使用数字，能看出更多信息。
