Kafka使用Apache Avro序列化

96
SimpleEasy
0.5 2019.04.05 16:05 字数 1161

1,什么是序列化和反序列化

当需要将数据存入文件或者通过网络发送出去时,需将数据对象转化为字节流,即对数据序列化,而将字节流还原成对象即反序列化。常用的序列化框架有Apache Avro、Twitter的Thrift和Google的Protobuf。它们之间的显著区别就是Avro支持IDL(接口描述语言)和Json描述schema,而Thrift和Protobuf只支持IDL描述Schema。本文将讨论Kafka中如何利用Apache avro框架序列/反序列化。

2,Apache avro序列化步骤

第一步:准备jar包

从官网下载avro-1.7.7.jar 和 avro-tools-1.7.7.jar两个jar包,放到指定文件目录。下载地址为http://www.trieuvan.com/apache/avro/avro-1.7.7/java/。

第二步:编写Schema描述文件

可以用Json和IDL描述schema,本文以Json描述schema文件。如创建stock.avsc的文本文件,文件内容如下:

{

    "namespace": "example.avro",

    "type": "record",

    "name": "Stock",

    "fields": [

        {"name": "stockCode", "type": "string"},

        {"name": "stockName",  "type": "string"},

        {"name": "tradeTime", "type": "long"},

        {"name": "preClosePrice", "type": "float"},

        {"name": "openPrice", "type": "float"},

        {"name": "currentPrice", "type": "float"},

        {"name": "highPrice", "type": "float"},

        {"name": "lowPrice", "type": "float"}

    ]

}

第三步:自动生成Java 对象类

执行如下命令:

java -jar avro-tools-1.7.7.jar compile schema stock.avsc java .

文中的 . 表示当前目录

第四步:在maven工程中引入下面jar包

<dependency>

  <groupId>org.apache.avro</groupId>

  <artifactId>avro</artifactId>

  <version>1.7.7</version>

</dependency>

第五步:将自动生成的java 对象源码放入到工程中,并注意修改代码中的包路径等。可以利用查找example.avro 替换成自己工程的包路径;

第六步:自定义序列化和反序列化类

序列化类:

package com.bigdata.kafkasender;

import java.io.ByteArrayOutputStream;

import java.io.IOException;

import java.util.Map;

import org.apache.avro.io.BinaryEncoder;

import org.apache.avro.io.DatumWriter;

import org.apache.avro.io.EncoderFactory;

import org.apache.avro.specific.SpecificDatumWriter;

import org.apache.kafka.common.errors.SerializationException;

import org.apache.kafka.common.serialization.Serializer;

public class AvroSerializer implements Serializer<Stock> {

    @Override

    public void close() {

    }

    @Override

    public void configure(Map<String, ?> arg0, boolean arg1) {

    }

    @Override

    public byte[] serialize(String topic, Stock data) {

        if (data == null) {

            return null;

        }

        DatumWriter<Stock> writer = new SpecificDatumWriter<>(data.getSchema());

        ByteArrayOutputStream out = new ByteArrayOutputStream();

        BinaryEncoder encoder = EncoderFactory.get().directBinaryEncoder(out, null);

        try {

            writer.write(data, encoder);

        } catch (IOException e) {

            throw new SerializationException(e.getMessage());

        }

        return out.toByteArray();

    }

}

反序列化类:

package com.bigdata.kafkasender;

import java.io.ByteArrayInputStream;

import java.io.IOException;

import java.util.Map;

import org.apache.avro.io.BinaryDecoder;

import org.apache.avro.io.DatumReader;

import org.apache.avro.io.DecoderFactory;

import org.apache.avro.specific.SpecificDatumReader;

import org.apache.kafka.common.serialization.Deserializer;

public class AvroDeserializer implements Deserializer<Stock> {

    @Override

    public void close() {}

    @Override

    public void configure(Map<String, ?> arg0, boolean arg1) {}

    @Override

    public Stock deserialize(String topic, byte[] data) {

        if(data == null) {

            return null;

        }

        Stock stock = new Stock();

        ByteArrayInputStream in = new ByteArrayInputStream(data);

        DatumReader<Stock> userDatumReader = new SpecificDatumReader<>(stock.getSchema());

        BinaryDecoder decoder = DecoderFactory.get().directBinaryDecoder(in, null);

        try {

            stock = userDatumReader.read(null, decoder);

        } catch (IOException e) {

            e.printStackTrace();

        }

        return stock;

    }

}

第七步:编写kafka生产者

package com.bigdata.kafkasender;

import org.apache.kafka.clients.producer.KafkaProducer;

import org.apache.kafka.clients.producer.ProducerRecord;

import org.apache.kafka.clients.producer.RecordMetadata;

import java.util.Properties;

import java.util.concurrent.ExecutionException;

import java.util.concurrent.Future;

public class KafkaSenderApplication {

    public static void main(String[] args) throws ExecutionException, InterruptedException {

        Stock[] stocks = new Stock[100];

        for (int i = 0; i < 100; i++) {

            stocks[i] = new Stock();

            stocks[i].setStockCode(String.valueOf(i));

            stocks[i].setStockName("stock" + i);

            stocks[i].setTradeTime(System.currentTimeMillis());

            stocks[i].setPreClosePrice(100.0F);

            stocks[i].setOpenPrice(88.8F);

            stocks[i].setCurrentPrice(120.5F);

            stocks[i].setHighPrice(300.0F);

            stocks[i].setLowPrice(12.4F);

        }

        Properties props = new Properties();

        props.put("bootstrap.servers", "xx.x.x.xx:9094");

        props.put("key.serializer", "org.apache.kafka.common.serialization.StringSerializer");

        props.put("value.serializer", "com.bigdata.kafkasender.AvroSerializer");

        KafkaProducer<String, Stock> producer = new KafkaProducer<>(props);

        Future<RecordMetadata> result = null;

        for (Stock stock : stocks) {

            ProducerRecord<String, Stock> record = new ProducerRecord<>("avro1", stock);

            RecordMetadata metadata = producer.send(record).get();

            StringBuilder sb = new StringBuilder();

            sb.append("stock: ").append(stock.toString()).append(" has been sent successfully!").append("\n")

                    .append("send to partition ").append(metadata.partition())

                    .append(", offset = ").append(metadata.offset());

            System.out.println(sb.toString());

            Thread.sleep(100);

        }

        producer.close();

    }

}

第八步:编写消费者

public static void consumeMessage() {

        Properties props = new Properties();

        /* 定义kakfa 服务的地址,不需要将所有broker指定上 */

        props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, "sxxxx:9094");

        /* 制定consumer group */

        // props.put("group.id", "flume");

        props.put("group.id", "avro1");

        props.put("auto.offset.reset", "earliest");

        /* key的序列化类 */

        props.put("key.deserializer", "org.apache.kafka.common.serialization.StringDeserializer");

        /* value的序列化类 */

        props.put("value.deserializer", "com.bigdata.kafkasender.AvroDeserializer");

        /* 定义consumer */

        KafkaConsumer<String, Stock> consumer = new KafkaConsumer<>(props);

        /* 消费者订阅的topic, 可同时订阅多个 */

        consumer.subscribe(Arrays.asList("avro1"));

        System.out.println("begin consume!");

        /* 读取数据,读取超时时间为100ms */

        Duration duration = Duration.ofMillis(100);

        while (true) {

            ConsumerRecords<String, Stock> records = consumer.poll(duration);

            for (ConsumerRecord<String, Stock> record : records)

                System.out.printf("offset = %d, key = %s, value = %s", record.offset(), record.key(),

                        record.value().toString() + "\n");

        }

    }

    public static void main(String[] args) {

// produceMessage();

        consumeMessage();

    }

随笔
Gupao