简书入口用户简单分析(二)——用户动态

字数 659阅读 809

在本人主观定义的入口用户(见简书入口用户简单分析)中选取了8143名用户,爬取他们的时间线,获取全部动态进行可视化分析

  • 所有动态的年月变化趋势,喜欢文章、发表评论和赞赏文章的趋势基本一致,可以看出简书身为创作者社区,发文与阅读的比例还是很高的:



    赞赏文章的行为在今年三月份到达高峰,16年中下半年是用户入住简书的高峰:



    纵轴对数分布:
  • 所有动态的一天24小时分布,一天的活动从早上6-8点开始,晚上9-11点有明显的阅读活动和发表文章的小高峰:


  • 所有动态的星期分布,差异不大:



    纵轴放大:



    换一种看法,动态在一周内的占比分布,星期六有个较明显的下降趋势,但还是区别不大:
  • 所有动态以月为周期按日取和统计,月末有个小下降,但31号的异常是因为某些月份没有31号:



    按动态的当日占比来看,相当均衡:


  • 几个最活跃用户(动态总数最多)的行为时间分布,作者梅话三弄和云儿飘过也分别是给别的作者打赏次数第一、第二多(1300多次打赏)的用户:





  • 给别人打赏第三多的用户



    用户打赏次数分布,差异还是相当大的,但是有过打赏行为的用户占了34.4%,还是很高的比例了。


  • 所有用户总活跃天数(一天内有动态该天即为活跃)分布,分布并不均衡,是不是老用户的活跃天数就多呢?



    将用户按入住年月分组,对他们的活跃天数取平均,可以看出16年左右入住的用户的活跃天数最多,并非是越老的用户活跃天数就越多,老用户也可能渐渐失去对平台的兴趣而流失。



    活跃比率(活跃天数除以入住总天数)的分布,大部分用户小于一周一次活跃:

    用户的简书入住总天数(横轴)与活跃比率(纵轴)的相关性分析,老用户的活跃度要低些,新用户正是热情高涨。


代码

# -------------------续简书入口用户简单分析代码---------------------------- #
# 获取动态
# share_note 发表文章
# like_comment 赞了评论
# like_note 喜欢了文章
# comment_note 发表评论
# like_collection 关注了专题
# reward_note 赞赏文章
# like_user 关注作者
# join_jianshu 加入简书
# like_notebook 喜欢专辑

# 数据库准备
from pymongo import MongoClient
client = MongoClient('localhost', 27017)
db = client.jianshu

# 辅助装饰器
from functools import wraps
def store_and_record_error(errors, coll):
    def decorator(f):
        @wraps(f)
        def wrapper(*args, **kwargs):
            try:
                # func_args = inspect.getcallargs(f, *args, **kwargs)
                res = f(*args, **kwargs)
                coll.insert_many(res)
                # return res
                # del res # 直接返回res内存占用过大
                return None
            except:
                errors.append(args[0])
                print('error', args[0])
                return None # []
        return wrapper
    return decorator

import re
errors = []
coll = db.user_active_new
@store_and_record_error(errors, coll)
@retry(Exception, delay=1, backoff=2, tries=2)
def get_active_from_single_user(id='45a15c9b5a22'):
    first_url = host + '/users/{id}/timeline?_pjax=%23list-container'.format(id=id)
    res = requests.get(first_url, headers=headers)

    # timetuple = time.strptime('2017-07-08T08:36:25', '%Y-%m-%dT%H:%M:%S')
    # datetime.datetime(*timetuple[0:6])

    infos = [] # 该用户全部动态
    match = re.compile('<li id="feed-(\d+)">')
    num = 2

    while 1:
        soup = BeautifulSoup(res.text, 'lxml')
        info = [{'id': id, # 多条动态  # todo:考虑内存占用问题
                 'action_time': i['data-datetime'].split('+')[0],
                 'action_type': i['data-type'],}
                for i in soup.select('.content .author .name span')]
        if not info:
            print('over', id)
            break
        infos.extend(info)
        max_id = int(re.findall(match, res.text)[-1]) - 1
        next_url = host + '/users/{id}/timeline?page={page_num}&max_id={max_id}'.format(
            id=id, max_id=max_id, page_num=num)
        res = requests.get(next_url, headers=headers)
        # print(num)
        num += 1

    return infos

# 4000个75分钟
# pool = Pool(30)
# iter map 暂不需要
# https://stackoverflow.com/questions/28375508/python-multiprocessing-tracking-the-process-of-pool-map-operation
# https://stackoverflow.com/questions/34827250/how-to-keep-track-of-status-with-multiprocessing-and-pool-map
# https://stackoverflow.com/questions/26520781/multiprocessing-pool-whats-the-difference-between-map-async-and-imap
# 首先要有users变量,即之前爬取的入口用户
all_active = pool.map(get_active_from_single_user, users['id'].values)
# Mongo Command: db.getCollection('user_active_new').distinct('id')

import pandas as pd
def read_data_from_mongo(coll):
    cursor = coll.find({}, {'_id': False})
    all_active = pd.DataFrame(list(cursor))#, dtype=int
    all_active.action_time = pd.to_datetime(all_active.action_time)
    all_active.action_type = all_active.action_type.astype('category')
    # 查看具体数值:all_active.action_type.cat.codes
    all_active.id = all_active.id.astype('category')
    all_active.info(memory_usage='deep')
    return all_active


coll = db.user_active_new
all_active = read_data_from_mongo(coll)

# -----------------------检查内存占用 (--------------------------- #
import sys  # dir() /globals() /locals() /vars() /whos
for var, obj in locals().items():
    print(var, sys.getsizeof(obj))
print('内存占用%sM' % (sys.getsizeof(all_active)/1048576))

import os, psutil
process = psutil.Process(os.getpid())
process.memory_info()[0] / float(2 ** 20)
process.memory_percent()
# -----------------------检查内存占用 )--------------------------- #


# 数据分析
id_active_times = all_active.groupby('id').size().to_frame(name='active_times').reset_index()
id_active_times.sort_values('active_times', inplace=True, ascending=False)
id_active_times.active_times.max()

weekday = all_active.action_time.apply(lambda x: x.isoweekday())
all_active['weekday'] = weekday
weekday = all_active.groupby('weekday').size()
weekday.plot.bar()
plt.show()

hours = all_active.action_time.apply(lambda x: x.hour)
all_active['hours'] = hours
hours = all_active.groupby('hours').size()
hours.plot.bar()
plt.show()

all_active = pd.concat([
    all_active,  
    pd.get_dummies(all_active['action_type'])], axis=1)
    
# all_active.groupby('weekday')['share_note'].sum()
# 分别画柱状图
data = all_active.groupby('weekday')
for d, t in zip(
    ['share_note', 'like_comment', 'like_note', 'comment_note', 'like_collection', 'reward_note',
     'like_user', 'join_jianshu', 'like_notebook'],
    ['发表文章', '赞了评论', '喜欢了文章', '发表评论', '关注了专题', '赞赏文章',
    '关注作者', '加入简书','喜欢专辑',]):
    temp = data[d].sum()
    fig = {
        'data': [go.Bar(x=temp.index.values,
                        y=temp.values,)],
        'layout': {'yaxis': {'title': t}},
    }
    plotly.offline.plot(fig, filename='basic_bar_%s.html'%d, show_link=False)

# 合在一起画柱状图
x = []
for d, t in zip(
    ['share_note', 'like_comment', 'like_note', 'comment_note', 'like_collection', 'reward_note',
     'like_user', 'join_jianshu', 'like_notebook'],
    ['发表文章', '赞了评论', '喜欢了文章', '发表评论', '关注了专题', '赞赏文章',
    '关注作者', '加入简书','喜欢专辑',]):
    temp = data[d].sum()
    x.append(go.Bar(x=temp.index.values,
                    y=temp.values,
                    name=t))
plotly.offline.plot(x, filename='basic_bar.html', show_link=False)

# 折线图
x = []
for d, t in zip(
    ['share_note', 'like_comment', 'like_note', 'comment_note', 'like_collection', 'reward_note',
     'like_user', 'join_jianshu', 'like_notebook'],
    ['发表文章', '赞了评论', '喜欢了文章', '发表评论', '关注了专题', '赞赏文章',
    '关注作者', '加入简书','喜欢专辑',]):
    temp = data[d].sum()
    # 是否看占比
    temp = temp / temp.sum()
    x.append(go.Scatter(x=temp.index.values,
                        y=temp.values,
                        mode = 'lines',
                        name = t))
plotly.offline.plot(x, filename='basic_bar.html', show_link=False)

# 各种行为按hour统计
data = all_active.groupby('hours')
x = []
for d, t in zip(
    ['share_note', 'like_comment', 'like_note', 'comment_note', 'like_collection', 'reward_note',
     'like_user', 'join_jianshu', 'like_notebook'],
    ['发表文章', '赞了评论', '喜欢了文章', '发表评论', '关注了专题', '赞赏文章',
    '关注作者', '加入简书','喜欢专辑',]):
    temp = data[d].sum()
    # temp = temp / temp.sum()
    x.append(go.Scatter(x=temp.index.values,
                        y=temp.values,
                        mode = 'lines',
                        name = t))
plotly.offline.plot(x, filename='basic_bar.html', show_link=False)

# 各种行为按day统计
all_active['day'] = all_active.action_time.apply(lambda x: x.day)
data = all_active.groupby('day')
x = []
for d, t in zip(
    ['share_note', 'like_comment', 'like_note', 'comment_note', 'like_collection', 'reward_note',
     'like_user', 'join_jianshu', 'like_notebook'],
    ['发表文章', '赞了评论', '喜欢了文章', '发表评论', '关注了专题', '赞赏文章',
    '关注作者', '加入简书','喜欢专辑',]):
    temp = data[d].mean()
    # temp = temp / temp.sum()
    x.append(go.Scatter(x=temp.index.values,
                        y=temp.values,
                        mode = 'lines+markers',
                        name = t))
plotly.offline.plot(x, filename='basic_bar.html', show_link=False)


# 加入简书的行为按年月统计
join_jianshu = all_active[all_active['action_type'] == 'join_jianshu']
# test: a = pd.datetime.strptime('2014-10-09 11:34:45', '%Y-%m-%d %H:%M:%S')
join_jianshu['year_month'] = join_jianshu['action_time'].dt.strftime('%Y-%m')
data = join_jianshu.groupby('year_month').size()

plotly.offline.plot([go.Scatter(
    x = data.index.values,
    y = data.values,
    mode = 'lines+markers',
    name = 'lines'
)], filename='lineZ.html', show_link=False)

# 所有行为按年月统计
all_active['year_month'] = all_active['action_time'].dt.strftime('%Y-%m')
# bug: categories is not json serializable:
# all_active['year_month'] = all_active['year_month'].astype('category')
data = all_active.groupby('year_month')
x = []
for d, t in zip(
    ['share_note', 'like_comment', 'like_note', 'comment_note', 'like_collection', 'reward_note',
     'like_user', 'join_jianshu', 'like_notebook'],
    ['发表文章', '赞了评论', '喜欢了文章', '发表评论', '关注了专题', '赞赏文章',
    '关注作者', '加入简书','喜欢专辑',]):
    temp = data[d].sum()
    x.append(go.Scatter(x=temp.index.values,
                        y=temp.values,
                        mode = 'lines+markers',
                        name = t))
fig = {'data': x,
       # 是否取对数坐标
       # 'layout': {'xaxis': {'title': '年月'}, 'yaxis': {'type': 'log'}}
}
plotly.offline.plot(fig, filename='basic_bar.html', show_link=False)


# todo: 最后一个月发表文章的占比

# 找出最活跃用户并按年月统计
most_active_user = all_active.groupby('id').size().sort_values(ascending=False).index[:5]
for mu in most_active_user:
    data = all_active[all_active['id'] == mu].groupby('year_month')
    x = []
    for d, t in zip(
        ['share_note', 'like_comment', 'like_note', 'comment_note', 'like_collection', 'reward_note',
         'like_user', 'join_jianshu', 'like_notebook'],
        ['发表文章', '赞了评论', '喜欢了文章', '发表评论', '关注了专题', '赞赏文章',
        '关注作者', '加入简书','喜欢专辑',]):
        temp = data[d].sum() # .mean() # 注意这里的含义
        x.append(go.Scatter(x=temp.index.values,
                            y=temp.values,
                            mode='lines+markers',
                            name=t))
    fig = {'data': x,
           # 是否取对数坐标
           # 'layout': {'xaxis': {'title': '年月'}, 'yaxis': {'type': 'log'}}
    }
    plotly.offline.plot(fig, filename='most_active_user_%s.html'%mu, show_link=False)

# 用户注册后的活跃天数占比
all_active['just_date'] = all_active.action_time.dt.date
active_days = all_active.groupby('id')['just_date'].apply(lambda x: x.nunique())
active_days.sort_values(ascending=False, inplace=True)
# - - 用户总活跃天数(一天内有动态该天即为活跃)分布
plotly.offline.plot([go.Bar(y=active_days)], filename='active_days.html', show_link=False)
# 算出从注册至今的总天数,然后与总活跃天数相比
now = pd.datetime.now()
join_during_time = all_active[
    all_active['action_type'] == 'join_jianshu'
][['id', 'action_time']]
join_during_time['during_time'] = now - join_during_time['action_time']
join_during_time['during_time'] = join_during_time.during_time.dt.days + 1
join_during_time = pd.merge(join_during_time,
    active_days.reset_index(name='active_days'))
join_during_time['ratio'] = join_during_time['active_days'] / join_during_time['during_time']
# 饼图
labels = ['小于10%', '大于10%小于50%', '大于50%小于90%', '大于90%']
ratio = join_during_time['ratio']
values = [(ratio > i).sum() for i in [0.1, 0.5, 0.9]]
values[1] -= values[2]
values.insert(1, len(ratio)-sum(values))
trace = go.Pie(labels=labels, values=values)
plotly.offline.plot([trace], filename='active_days.html', show_link=False)
# 直方图与相关性分布图
join_during_time['ratio'].hist()
sns.jointplot(data=join_during_time, x='during_time', y='ratio', kind='reg', color='g')
sns.plt.show()

# 按注册年月显示的平均活跃天数
join_during_time['year_month'] = join_during_time['action_time'].dt.strftime('%Y-%m')
temp = join_during_time.groupby('year_month')['active_days'].mean()
plotly.offline.plot([go.Scatter(x=temp.index.values,
                               y=temp.values,
                               mode = 'lines+markers')],
                    filename='average_active_days.html',
                    show_link=False)

# 打赏次数最多的用户的行为
most_reward_user = all_active.groupby('id')['reward_note'].sum().sort_values(ascending=False).index[2]
data = all_active[all_active['id'] == most_reward_user].groupby('year_month')
x = []
for d, t in zip(
    ['share_note', 'like_comment', 'like_note', 'comment_note', 'like_collection', 'reward_note',
     'like_user', 'join_jianshu', 'like_notebook'],
    ['发表文章', '赞了评论', '喜欢了文章', '发表评论', '关注了专题', '赞赏文章',
    '关注作者', '加入简书','喜欢专辑',]):
    temp = data[d].sum()
    x.append(go.Scatter(x=temp.index.values,
                        y=temp.values,
                        mode = 'lines+markers',
                        name = t))
fig = {'data': x,
       # 是否取对数坐标
       # 'layout': {'xaxis': {'title': '年月'}, 'yaxis': {'type': 'log'}}
}
plotly.offline.plot(fig, filename='most_reward_user_%s.html'%most_reward_user, show_link=False)

# 打赏用户占比
reward = all_active.groupby('id')['reward_note'].sum().sort_values(ascending=False)
print((reward > 0).sum() / len(reward))
plotly.offline.plot([go.Bar(y=reward)], filename='reward.html', show_link=False)

其他

  • 热门文章爬取(前两页)
def get_hot_articles_from_single_user(id='45a15c9b5a22'):
    hot_first_url = host + '/u/{id}?order_by=top&_pjax=%23list-container'.format(id=id)
    hot_next_url = host + '/u/{id}?order_by=top&page={page_num}'.format(id=id, page_num=2)
    user_hot_articles = []
    for u in [hot_first_url, hot_next_url]:
        res = requests.get(u, headers=headers)
        soup = BeautifulSoup(res.text, 'lxml')
        for info in soup.select('.content'):
            title = info.select_one('.title').text
            time_ = info.select_one('.time')['data-shared-at']
            details = info.select_one('.meta')
            read, comments = [i.text.strip() for i in details.findAll('a')]
            like_money = [i.text.strip() for i in details.findAll('span')]
            if len(like_money) == 1:
                like = like_money[0]
                money = 0
            else:
                like, money = like_money
            user_hot_articles.append({
                'title': title,
                'time': time_,
                'read': read,
                'comments': comments,
                'like': like,
                'money': money,
            }) 

推荐阅读更多精彩内容