参考书籍：《spark最佳实践》

实验步骤：
1.Python程序生成访问日志
2.通过脚本将日志自动上传至HDFS
3.spark streaming程序监控HDFS目录，自动处理新的文件。

log.py文件代码：

import random
import time

class WebLogGeneration(object):
    def __init__(self):
        self.user_agent_dist={
            0.0:"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)",
            0.1:"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Trident/6.0)",
            0.2:"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0;.NET CLR 2.0.50727)",
            0.3:"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)",
            0.4:"Mozilla/5.0 (Windows NT 6.1; Trident/7.0; rv:11.0) like Gecko",
            0.5:"Mozilla/5.0 (Windows NT 6.1; WOW64;rv:41.0) Gecko/20100101 Firefox/41.0",
            0.6:"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.0; .NET CLR 1.1.4322)",
            0.7:"Mozilla/5.0 (iPhone; CPU iPhone OS 7_0_3 like Mac OS X) AppleWebKit/537.51.1 (KHTML,like Gecko) Version/7.0 Mobile/11B511 Safari/9537.53",
            0.8:"Mozilla/5.0 (Linux; Android 4.2.1; Galaxy Nexus Build/JOP40D) AppleWebKit/535.19 (KHTML,like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19",
            0.9:"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML,like Gecko) Chrome/45.0.2454.85 Safari/537.36",
            1:" "
        }
        self.ip_slice_list = [10,29,30,46,55,63,72,87,98,132,156,124,1677,143,187,168,190,201,202,214,215,222]
        self.url_path_list = ["login.php","view.php","list.php","upload.php","admin/login.php","edit.php","index.html"]
        self.http_refer = [
            "http://www.baidu.com/s?wd={query}",
            "http://www.google.cn/search?q={query}",
            "http://www.sogou.com/web?query={query}",
            "http://www.yahoo.com/s?p={query}",
            "http://cn.bing.com/search?q={query}"
        ]
        self.search_keyword = ["spark","hadoop","hive","spark mlib","spark sql"]

    def sample_ip(self):
        slice = random.sample(self.ip_slice_list, 4)
        return ".".join(str(item) for item in slice)

    def sample_url(self):
        return random.sample(self.url_path_list,1)[0]

    def sample_user_agent(self):
        dist_uppon = random.uniform(0, 1)
        return self.user_agent_dist[float('%0.1f' % dist_uppon)]

    def sample_refer(self):
        if random.uniform(0, 1) > 0.2:
            return "-"
        refer_str = random.sample(self.http_refer, 1)
        query_str = random.sample(self.search_keyword, 1)
        return refer_str[0].format(query=query_str)

    def sample_one_log(self,count = 3):
        time_str = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())
        while count > 1:
            query_log = "{ip} - - [{local_time}] \"GET /{url} HTTP/1.1\" 200 0 \"{refer}\" \"{user_agent}\" \"-\"".format(ip=self.sample_ip(),local_time=time_str,url=self.sample_url(),refer=self.sample_refer(),user_agent=self.sample_user_agent())
            print query_log
            count = count - 1

if __name__ == "__main__":
    web_log_gene = WebLogGeneration()
    web_log_gene.sample_one_log(random.uniform(1000,2000))

run.sh脚本代码：

#!/bin/bash

streaming_dir="/spark/streaming"
while [ 1 ];do
    python log.py >> test.log
    tmplog="access.`date +'%s'`.log"
    hadoop fs -put test.log ${streaming_dir}/tmp/$tmplog
    hadoop fs -mv ${streaming_dir}/tmp/$tmplog ${streaming_dir}/
    echo "`date +"%F %T"` put $tmplog to HDFS succeed"
    sleep 1
done

SparkStreamingTest.scala文件代码：

package spark.example
import org.apache.spark.SparkConf
import org.apache.spark.Streaming.{Seconds,streamingContext}

object SparkStreamingTest{
    def main(args:Array[String]){
        val batch = 10

        val conf = new SparkConf().setAppName("NginxAnay")
        val ssc = new streamingContext(conf,Seconds(batch))

        val lines = ssc.textFileStream("hdfs://7master:9000/spark/streaming")

        // 1.总PV
        lines.count().print()

        // 2.各IP的PV，按PV倒叙
        // 空格分隔的第一个字段就是IP
        lines.map(line =>{(line.split(" ")(0), 1)}).reduceByKey(_ + _).transform(rdd =>{
            rdd.map(ip_pv => (ip_pv._2, ip_pv._1)).sortByKey(false).map(ip_pv => (ip_pv._2,ip_pv._1))
        }).print()

        // 3.搜索引擎PV
        // 先输出搜索引擎和查询关键词，避免统计搜索关键词时重复计算
        // 输出(host, query_keys)
        val refer = lines.map(_.split("\"")(3))
        val searchEnginInfo = refer.map(r =>{
            val f = r.split('/')
            val searchEngines = Map(
                "www.google.cn" -> "q",
                "www.yahoo.com" -> "p",
                "cn.bing.com" -> "q",
                "www.baidu.com" -> "wd",
                "www.sogou.com" -> "query"
            )
            if(f.length > 2){
                val host = f(2)
                if(searchEngines.contains(host)){
                    val query = r.split('?')(1)
                    if(query.length > 0){
                        val arr_search_q = query.split('&').filter(_.indexOf(searchEngines(host)+"=") == 0)
                        if(arr_search_q.length > 0){
                            (host,arr_search_q(0).split('=')(1))
                        }else{
                            (host, "")
                        }
                    }else{
                        (host, "")
                    }
                }else{
                    ("", "")
                }
            }else{
                ("", "")
            }
        })

        // 输出搜索引擎PV
        searchEnginInfo.filter(_._1.length > 0).map(p => {(p._1, 1)}).reduceByKey(_ + _).print()

        // 4.关键词PV
        searchEnginInfo.filter(_._2.length > 0).map(p => {(p._2, 1)}).reduceByKey(_ + _).print()

        // 5.终端类型PV
        lines.map(_.split("\"")(5)).map(agent =>{
            val types = Seq("iPhone", "Android")
            var r = "Default"
            for (t <- types){
                if(agent.indexOf(t) != -1)
                    r = t
            }
            (r, 1)
        }).reduceByKey(_ + _).print()

        // 6.各页面PV
        lines.map(line => {
            (line.split("\"")(1).split(" ")(1), 1)
        }).reduceByKey(_ + _).print()

        // 启动计算，等待执行结束
        ssc.start()
        ssc.awaitTermination()
    }
}

run_SparkStreamingTest.sh文件代码：

/usr/local/src/spark-1.6.0-bin-hadoop2.6/bin/spark-submit --master local[2] \
--class spark.example.SparkStreamingTest target/scala-2.11/wordcount_2.11-1.6.0.jar