上一篇文章的地址是：http://blog.csdn.net/callmesp/article/details/74372582

这次带来的是一个 当当网 的爬虫app。
先放出GitHub链接：

https://github.com/CallMeSp/DangDang

先看一下结构分析图：

这里写图片描述

功能不复杂，但是有Activity、Adapter、自定义view、MVP结构、SQLite等一个app有的几乎一切。
功能实现过程说明：

• 在搜索框中输入要搜索的书籍名称

• 点击搜索后，首先判断数据库中是否有缓存

• 有缓存则默认加载数据库中的缓存

• 否则通过Jsoup去爬取

• 爬取的结果会存入新建的一张对应搜索名字的表

• 上拉加载更多：会将新加载的书籍列表存入对应数据库

• 下拉刷新：将已有的数据库中的缓存清空，重新爬取，并将数据写入数据库

下面进入正文：
首先这是一个爬虫，要先建立我们的model类

class Bookitem (var map:MutableMap<String,Any?>){
    var name: String by map

    var detail: String by map

    var price: String by map

    var author: String by map

    var imgurl: String by map

    var publisher: String by map

    var detailurl:String by map
    constructor(name :String,detail:String,price:String,author:String,imgurl:String,publisher:String,detailurl:String):this(HashMap()){
        this.name=name
        this.detail=detail
        this.price=price
        this.author=author
        this.imgurl=imgurl
        this.publisher=publisher
        this.detailurl=detailurl
    }
}

本来单纯建立这样一个类是并不需要这个Map的，但是这里面是为了后面的Sqlite操作做铺垫，anko框架中数据库存储和读取值得时候的映射就是要通过这个Map来实现的，要注意的是这里面每个变量的名字要和数据库Table中对应的列的名字一样。

createTable(tbname,true,"id" to INTEGER+ PRIMARY_KEY+ AUTOINCREMENT,
        "name" to TEXT,
        "price" to TEXT,
        "author" to TEXT,
        "publisher" to TEXT,
        "detail" to TEXT,
        "imgurl" to TEXT,
        "detailurl" to TEXT)

下面就来看一下关键的几个功能实现：
1.根据书名和页码获取列表

 fun getBookListByNameAndPage(bookname: String, page: Int, ismore: Boolean) {
    Observable.just(bookname)
            .observeOn(Schedulers.newThread())
            .map { s ->
                mainpresenter.showPB()
                val bookitemArrayList = ArrayList<Bookitem>()
                val doc = Jsoup.connect("http://search.dangdang.com/?key=$s&act=input&page_index=$page").get()
                Log.e("..",doc.toString())
                val elements = doc.select("ul.bigimg").select("li")
                for (element in elements) {
                    val name = element.select("a").attr("title")
                    val detailurl = element.select("a").attr("href")
                    val imgurl = if (!element.select("a").select("img").attr("src").startsWith("http"))
                        element.select("a").select("img").attr("data-original")
                    else
                        element.select("a").select("img").attr("src")
                    val detail = element.select("p.detail").text()
                    val price = element.select("p.price").select("span.search_now_price").text()
                    val author = element.select("p.search_book_author").select("span")[0].select("a").attr("title")
                    val publisher = element.select("p.search_book_author").select("span")[2].select("a").text()
                    val bookitem = Bookitem(name ,detail,price,author,imgurl,publisher,detailurl)
                    Log.e(TAG,"detailurl:"+detailurl)
                    bookitemArrayList.add(bookitem)
                }
                bookitemArrayList
            }
            .subscribe(object : DisposableObserver<ArrayList<Bookitem>>() {
                override fun onNext(bookitems: ArrayList<Bookitem>) {
                    if (!ismore) {
                        mainpresenter.updateUI(bookitems)
                    } else {
                        mainpresenter.addMoreList(bookitems)
                    }
                }

                override fun onError(e: Throwable) {
                    mainpresenter.updateUI(ArrayList<Bookitem>())
                    mainpresenter.hidePB()
                }

                override fun onComplete() {
                    mainpresenter.hidePB()
                }
            })
}

2.书籍详情界面获取procontent

fun getDetail(url:String){
    Observable.just(url)
            .observeOn(Schedulers.newThread())
            .subscribe(object :DisposableObserver<String>(){
                override fun onError(e: Throwable?) {

                }

                override fun onComplete() {

                }

                override fun onNext(t: String?) {
                    val detailurl = url
                    Log.e(TAG, "url?:" + t)
                    

                    val doc = Jsoup.connect(detailurl).get()
                    Log.e("??", doc.toString())
                    var elements_content = doc.select("div.t_box").select("div.t_box_left")
                    var procontent = doc.select("div.t_box").select("div.t_box_left").select("div.pro_content").select("ul").select("li")
                    var contents = doc.select("div.section")
                    var catalog = doc.select("div.section")

                    var stb:StringBuffer= StringBuffer()
                    //procontent加入换行符
                    for (str:Element in procontent){
                        stb.append(str.text())
                        stb.append("\n")
                    }

                    detailpresenter.setProContent(stb.toString())
                    Log.e("procontent",stb.toString())
                }
            })

}

3.DbHelper

class DbHelper(context:Context):ManagedSQLiteOpenHelper(context, DB_NAME,null, DB_VERSION){
    val mcontext=context
    @Volatile var TB_NAME:String=""
    companion object {
        val DB_NAME="history.db"
        val DB_VERSION=1
        val TAG="DbHelper"
        //val instance by lazy{ DbHelper()}
    }
    override fun onCreate(db: SQLiteDatabase?) {
        Log.e(TAG,"onCreate")
    }

    fun insertNewTable(tbname:String,booklist:ArrayList<Bookitem>){
        DbHelper(mcontext).use {
            /*createTable(tbname,true,"id" to INTEGER+ PRIMARY_KEY+ AUTOINCREMENT,
                    "bookname" to TEXT,
                    "price" to TEXT,
                    "author" to TEXT,
                    "publisher" to TEXT,
                    "detail" to TEXT,
                    "imgurl" to TEXT,
                    "detailurl" to TEXT)*/
            execSQL("create table if not exists "+tbname+"(it INTEGER primary key AUTOINCREMENT, " +
                    "name text," +
                    "detail text," +
                    "price text," +
                    "author text,"
                    +"imgurl text," +
                    "publisher text," +
                    "detailurl text)")
            Log.e(TAG,"create table:"+tbname)
            booklist.forEach { insert(tbname,
                    "name" to it.name,
                    "price" to it.price,
                    "author" to it.author,
                    "publisher" to it.author,
                    "detail" to it.detail,
                    "imgurl" to it.imgurl,
                    "detailurl" to it.detailurl)
                Log.e(TAG,"insert into table:"+tbname+": "+it.name)}
        }
    }
    fun addToOldTable(tbname:String,booklist:ArrayList<Bookitem>) {
        DbHelper(mcontext).use {
            booklist.forEach {
                insert(tbname,
                        "name" to it.name,
                        "price" to it.price,
                        "author" to it.author,
                        "publisher" to it.author,
                        "detail" to it.detail,
                        "imgurl" to it.imgurl,
                        "detailurl" to it.detailurl)
                Log.e(TAG,"insert into table:"+tbname+": "+it.name)
            }
        }
    }

    fun getListFormDb(bookname:String)= DbHelper(mcontext).use {
        select(bookname).parseList {
            Bookitem(HashMap(it))
        }
    }

    fun dropTable(bookname: String)=DbHelper(mcontext).use {
        clear(bookname)
    }
    fun IshaveTB(bookname:String):Boolean=DbHelper(mcontext).use {
        var result:Boolean=false
        var cursor=rawQuery("select count(*) as c from sqlite_master where type ='table' and name ="+"\'"+bookname+"\';",null)
        if (cursor.moveToNext()){
            var count:Int=cursor.getInt(0)
            if (count>0){
                result=true
            }
        }
        result
    }
    override fun onUpgrade(db: SQLiteDatabase?, oldVersion: Int, newVersion: Int) {
        db!!.dropTable(TB_NAME,true)
        onCreate(db)
    }
    fun <T : Any> SelectQueryBuilder.parseList(parser: (Map<String, Any?>) -> T): List<T> =
            parseList(object : MapRowParser<T> {
                override fun parseRow(columns: Map<String, Any?>): T = parser(columns)
            })
    fun <T : Any> SelectQueryBuilder.parseOpt(parser: (Map<String, Any?>) -> T): T? =
            parseOpt(object : MapRowParser<T> {
                override fun parseRow(columns: Map<String, Any?>): T = parser(columns)
            })
    fun SQLiteDatabase.clear(tableName: String) {
        execSQL("delete from $tableName")
    }
}

4.上拉下拉监听：

class SwipeRefreshView(context: Context, attrs: AttributeSet) : SwipeRefreshLayout(context, attrs) {

    private val mScaledTouchSlop: Int

    private val mFooterView: View

    private var recyclerview: RecyclerView? = null

    @Volatile private var condition2: Boolean = false

    private var MonLoadListener:()->Unit={}
    /**
     * 正在加载状态
     */
    private var isLoading: Boolean = false

    internal var layoutInflater: LayoutInflater

    init {
        // 填充底部加载布局
        mFooterView = View.inflate(context, R.layout.view_footer, null)
        // 表示控件移动的最小距离，手移动的距离大于这个距离才能拖动控件
        mScaledTouchSlop = ViewConfiguration.get(context).scaledTouchSlop
        layoutInflater = LayoutInflater.from(context)
    }

    override fun onLayout(changed: Boolean, left: Int, top: Int, right: Int, bottom: Int) {
        super.onLayout(changed, left, top, right, bottom)
        // recyclerview,设置recyclerview的布局位置
        if (recyclerview == null) {
            // 判断容器有多少个孩子
            if (childCount > 0) {
                // 判断第一个孩子是不是ListView
                if (getChildAt(0) is RecyclerView) {
                    // 创建ListView对象
                    recyclerview = getChildAt(0) as RecyclerView

                    // 设置ListView的滑动监听
                    setRecyclerViewOnScroll()
                }
            }
        }
    }

    /**
     * 在分发事件的时候处理子控件的触摸事件

     * @param ev
     * *
     * @return
     */
    private var mDownY: Float = 0.toFloat()
    private var mUpY: Float = 0.toFloat()

    override fun dispatchTouchEvent(ev: MotionEvent): Boolean {

        when (ev.action) {
            MotionEvent.ACTION_DOWN ->
                // 移动的起点
                mDownY = ev.y
            MotionEvent.ACTION_MOVE ->
                // 移动过程中判断时候能下拉加载更多
                if (condition2 && canLoadMore()) {
                    // 加载数据
                    loadData()
                    return false
                }
            MotionEvent.ACTION_UP ->
                // 移动的终点
                mUpY = y
        }
        return super.dispatchTouchEvent(ev)
    }

    /**
     * 判断是否满足加载更多条件

     * @return
     */
    private fun canLoadMore(): Boolean {
        // 1. 是上拉状态
        val condition1 = mDownY - mUpY >= mScaledTouchSlop
        if (condition1) {
            println("是上拉状态")
        }

        // 3. 正在加载状态
        val condition3 = !isLoading
        if (condition3) {
            println("不是正在加载状态")
        }
        return condition1 && condition3
    }

    /**
     * 处理加载数据的逻辑
     */
    private fun loadData() {
        println("加载数据...")
        if (MonLoadListener != null) {
            // 设置加载状态，让布局显示出来
            setLoading(true)
            MonLoadListener()
        }
    }

    /**
     * 设置加载状态，是否加载传入boolean值进行判断

     * @param loading
     */
    fun setLoading(loading: Boolean) {
        // 修改当前的状态
        isLoading = loading
     }


    /**
     * 设置RecyclerView的滑动监听
     */
    private fun setRecyclerViewOnScroll() {

        recyclerview!!.setOnScrollChangeListener { v, scrollX, scrollY, oldScrollX, oldScrollY ->
            val layoutManager = recyclerview!!.layoutManager
            val linearManager = layoutManager as LinearLayoutManager
            val lastItemPosition = linearManager.findLastVisibleItemPosition()
            if (recyclerview!!.adapter.itemCount - 1 == lastItemPosition) {
                condition2 = true
            } else {
                condition2 = false
            }
        }
    }

    fun setCondition2(isLoading: Boolean) {
        condition2 = isLoading
    }


    /**
     * 上拉加载的接口回调
     */


    fun setMOnLoadListener(listener:()->Unit){
        MonLoadListener=listener
    }

    companion object {

        private val TAG = "SwipeRefreshView"
    }
}

要注意的是这里的DBhelper中的扩展函数。
如：

fun dropTable(bookname: String)=DbHelper(mcontext).use {
        clear(bookname)
    }

• Kotlin的扩展函数功能使得我们可以为现有的类添加新的函数，实现某一具体功能 。

• 扩展函数是静态解析的，并未对原类添加函数或属性，对类本身没有任何影响。

• 扩展属性允许定义在类或者kotlin文件中，不允许定义在函数中。